Anomaly detection API

Use these anomaly detection operations to programmatically create and manage detectors.

Create anomaly detector

Introduced 1.0

Creates an anomaly detector.

This command creates a single-entity detector named test-detector that finds anomalies based on the sum of the value field and stores the result in a custom opensearch-ad-plugin-result-test index:

Request

  1. POST _plugins/_anomaly_detection/detectors
  2. {
  3.   "name": "test-detector",
  4.   "description": "Test detector",
  5.   "time_field": "timestamp",
  6.   "indices": [
  7.     "server_log*"
  8.   ],
  9.   "feature_attributes": [
  10.     {
  11.       "feature_name": "test",
  12.       "feature_enabled": true,
  13.       "aggregation_query": {
  14.         "test": {
  15.           "sum": {
  16.             "field": "value"
  17.           }
  18.         }
  19.       }
  20.     }
  21.   ],
  22.   "filter_query": {
  23.     "bool": {
  24.       "filter": [
  25.         {
  26.           "range": {
  27.             "value": {
  28.               "gt": 1
  29.             }
  30.           }
  31.         }
  32.       ],
  33.       "adjust_pure_negative": true,
  34.       "boost": 1
  35.     }
  36.   },
  37.   "detection_interval": {
  38.     "period": {
  39.       "interval": 1,
  40.       "unit": "Minutes"
  41.     }
  42.   },
  43.   "window_delay": {
  44.     "period": {
  45.       "interval": 1,
  46.       "unit": "Minutes"
  47.     }
  48.   },
  49.   "result_index" : "opensearch-ad-plugin-result-test"
  50. }

Example response

  1. {
  2.   "_id": "VEHKTXwBwf_U8gjUXY2s",
  3.   "_version": 1,
  4.   "_seq_no": 5,
  5.   "anomaly_detector": {
  6.     "name": "test-detector",
  7.     "description": "Test detector",
  8.     "time_field": "timestamp",
  9.     "indices": [
  10.       "server_log*"
  11.     ],
  12.     "filter_query": {
  13.       "bool": {
  14.         "filter": [
  15.           {
  16.             "range": {
  17.               "value": {
  18.                 "from": 1,
  19.                 "to": null,
  20.                 "include_lower": false,
  21.                 "include_upper": true,
  22.                 "boost": 1
  23.               }
  24.             }
  25.           }
  26.         ],
  27.         "adjust_pure_negative": true,
  28.         "boost": 1
  29.       }
  30.     },
  31.     "detection_interval": {
  32.       "period": {
  33.         "interval": 1,
  34.         "unit": "Minutes"
  35.       }
  36.     },
  37.     "window_delay": {
  38.       "period": {
  39.         "interval": 1,
  40.         "unit": "Minutes"
  41.       }
  42.     },
  43.     "shingle_size": 8,
  44.     "schema_version": 0,
  45.     "feature_attributes": [
  46.       {
  47.         "feature_id": "U0HKTXwBwf_U8gjUXY2m",
  48.         "feature_name": "test",
  49.         "feature_enabled": true,
  50.         "aggregation_query": {
  51.           "test": {
  52.             "sum": {
  53.               "field": "value"
  54.             }
  55.           }
  56.         }
  57.       }
  58.     ],
  59.     "last_update_time": 1633392680364,
  60.     "user": {
  61.       "name": "admin",
  62.       "backend_roles": [
  63.         "admin"
  64.       ],
  65.       "roles": [
  66.         "own_index",
  67.         "all_access"
  68.       ],
  69.       "custom_attribute_names": [],
  70.       "user_requested_tenant": "__user__"
  71.     },
  72.     "detector_type": "SINGLE_ENTITY"
  73.   },
  74.   "_primary_term": 1
  75. }

To create a high cardinality detector by specifying a category field:

Request

  1. POST _plugins/_anomaly_detection/detectors
  2. {
  3.   "name": "test-hc-detector",
  4.   "description": "Test detector",
  5.   "time_field": "timestamp",
  6.   "indices": [
  7.     "server_log*"
  8.   ],
  9.   "feature_attributes": [
  10.     {
  11.       "feature_name": "test",
  12.       "feature_enabled": true,
  13.       "aggregation_query": {
  14.         "test": {
  15.           "sum": {
  16.             "field": "value"
  17.           }
  18.         }
  19.       }
  20.     }
  21.   ],
  22.   "filter_query": {
  23.     "bool": {
  24.       "filter": [
  25.         {
  26.           "range": {
  27.             "value": {
  28.               "gt": 1
  29.             }
  30.           }
  31.         }
  32.       ],
  33.       "adjust_pure_negative": true,
  34.       "boost": 1
  35.     }
  36.   },
  37.   "detection_interval": {
  38.     "period": {
  39.       "interval": 1,
  40.       "unit": "Minutes"
  41.     }
  42.   },
  43.   "window_delay": {
  44.     "period": {
  45.       "interval": 1,
  46.       "unit": "Minutes"
  47.     }
  48.   },
  49.   "category_field": [
  50.     "ip"
  51.   ]
  52. }

Example response

  1. {
  2.   "_id": "b0HRTXwBwf_U8gjUw43R",
  3.   "_version": 1,
  4.   "_seq_no": 6,
  5.   "anomaly_detector": {
  6.     "name": "test-hc-detector",
  7.     "description": "Test detector",
  8.     "time_field": "timestamp",
  9.     "indices": [
  10.       "server_log*"
  11.     ],
  12.     "filter_query": {
  13.       "bool": {
  14.         "filter": [
  15.           {
  16.             "range": {
  17.               "value": {
  18.                 "from": 1,
  19.                 "to": null,
  20.                 "include_lower": false,
  21.                 "include_upper": true,
  22.                 "boost": 1
  23.               }
  24.             }
  25.           }
  26.         ],
  27.         "adjust_pure_negative": true,
  28.         "boost": 1
  29.       }
  30.     },
  31.     "detection_interval": {
  32.       "period": {
  33.         "interval": 1,
  34.         "unit": "Minutes"
  35.       }
  36.     },
  37.     "window_delay": {
  38.       "period": {
  39.         "interval": 1,
  40.         "unit": "Minutes"
  41.       }
  42.     },
  43.     "shingle_size": 8,
  44.     "schema_version": 0,
  45.     "feature_attributes": [
  46.       {
  47.         "feature_id": "bkHRTXwBwf_U8gjUw43K",
  48.         "feature_name": "test",
  49.         "feature_enabled": true,
  50.         "aggregation_query": {
  51.           "test": {
  52.             "sum": {
  53.               "field": "value"
  54.             }
  55.           }
  56.         }
  57.       }
  58.     ],
  59.     "last_update_time": 1633393165265,
  60.     "category_field": [
  61.       "ip"
  62.     ],
  63.     "user": {
  64.       "name": "admin",
  65.       "backend_roles": [
  66.         "admin"
  67.       ],
  68.       "roles": [
  69.         "own_index",
  70.         "all_access"
  71.       ],
  72.       "custom_attribute_names": [],
  73.       "user_requested_tenant": "__user__"
  74.     },
  75.     "detector_type": "MULTI_ENTITY"
  76.   },
  77.   "_primary_term": 1
  78. }

You can specify a maximum of two category fields:

  1. "category_field": [
  2.   "ip"
  3. ]
  1. "category_field": [
  2.   "ip", "error_type"
  3. ]

You can specify the following options.

OptionsDescriptionTypeRequired
nameThe name of the detector.stringYes
descriptionA description of the detector.stringNo
time_fieldThe name of the time field.stringYes
indicesA list of indices to use as the data source.listYes
feature_attributesSpecify a feature_name, set the enabled parameter to true, and specify an aggregation query.listYes
filter_queryProvide an optional filter query for your feature.objectNo
detection_intervalThe time interval for your anomaly detector.objectYes
window_delayAdd extra processing time for data collection.objectNo
category_fieldCategorizes or slices data with a dimension. Similar to GROUP BY in SQL.listNo

Validate detector

Introduced 1.2

Returns whether the detector configuration has any issues that might prevent OpenSearch from creating the detector.

You can use the validate detector API operation to identify issues in your detector configuration before creating the detector.

The request body consists of the detector configuration and follows the same format as the request body of the create detector API.

You have the following validation options:

  • Only validate against the detector configuration and find any issues that would completely block detector creation:
  1. POST _plugins/_anomaly_detection/detectors/_validate
  2. POST _plugins/_anomaly_detection/detectors/_validate/detector
  • Validate against the source data to see how likely the detector would complete model training.
  1. POST _plugins/_anomaly_detection/detectors/_validate/model

Responses from this API operation return either blocking issues as detector type responses or a response indicating a field that could be revised to increase likelihood of model training completing successfully. Model type issues don’t need to be fixed for detector creation to succeed, but the detector would likely not train successfully if they aren’t addressed.

Request

  1. POST _plugins/_anomaly_detection/detectors/_validate
  2. POST _plugins/_anomaly_detection/detectors/_validate/detector
  3. {
  4.   "name": "test-detector",
  5.   "description": "Test detector",
  6.   "time_field": "timestamp",
  7.   "indices": [
  8.     "server_log*"
  9.   ],
  10.   "feature_attributes": [
  11.     {
  12.       "feature_name": "test",
  13.       "feature_enabled": true,
  14.       "aggregation_query": {
  15.         "test": {
  16.           "sum": {
  17.             "field": "value"
  18.           }
  19.         }
  20.       }
  21.     }
  22.   ],
  23.   "filter_query": {
  24.     "bool": {
  25.       "filter": [
  26.         {
  27.           "range": {
  28.             "value": {
  29.               "gt": 1
  30.             }
  31.           }
  32.         }
  33.       ],
  34.       "adjust_pure_negative": true,
  35.       "boost": 1
  36.     }
  37.   },
  38.   "detection_interval": {
  39.     "period": {
  40.       "interval": 1,
  41.       "unit": "Minutes"
  42.     }
  43.   },
  44.   "window_delay": {
  45.     "period": {
  46.       "interval": 1,
  47.       "unit": "Minutes"
  48.     }
  49.   }
  50. }

If the validate detector API doesn’t find any issue in the detector configuration, it returns an empty response:

Example response

  1. {}

If the validate detector API finds an issue, it returns a message explaining what’s wrong with the configuration. In this example, the feature query aggregates over a field that doesn’t exist in the data source:

Example response

  1. {
  2.   "detector": {
  3.     "feature_attributes": {
  4.       "message": "Feature has invalid query returning empty aggregated data: average_total_rev",
  5.       "sub_issues": {
  6.         "average_total_rev": "Feature has invalid query returning empty aggregated data"
  7.       }
  8.     }
  9.   }
  10. }

The following request validates against the source data to see if model training might succeed. In this example, the data is ingested at a rate of every 5 minutes, and detector interval is set to 1 minute.

  1. POST _plugins/_anomaly_detection/detectors/_validate/model
  2. {
  3.   "name": "test-detector",
  4.   "description": "Test detector",
  5.   "time_field": "timestamp",
  6.   "indices": [
  7.     "server_log*"
  8.   ],
  9.   "feature_attributes": [
  10.     {
  11.       "feature_name": "test",
  12.       "feature_enabled": true,
  13.       "aggregation_query": {
  14.         "test": {
  15.           "sum": {
  16.             "field": "value"
  17.           }
  18.         }
  19.       }
  20.     }
  21.   ],
  22.   "filter_query": {
  23.     "bool": {
  24.       "filter": [
  25.         {
  26.           "range": {
  27.             "value": {
  28.               "gt": 1
  29.             }
  30.           }
  31.         }
  32.       ],
  33.       "adjust_pure_negative": true,
  34.       "boost": 1
  35.     }
  36.   },
  37.   "detection_interval": {
  38.     "period": {
  39.       "interval": 1,
  40.       "unit": "Minutes"
  41.     }
  42.   },
  43.   "window_delay": {
  44.     "period": {
  45.       "interval": 1,
  46.       "unit": "Minutes"
  47.     }
  48.   }
  49. }

If the validate detector API finds areas of improvement with your configuration, it returns a response with suggestions about how you can change your configuration to improve model training.

Sample Responses

In this example, the validate detector API returns a response indicating that changing the detector interval length to at least four minutes can increase the chances of successful model training.

  1. {
  2.   "model": {
  3.     "detection_interval": {
  4.       "message": "The selected detector interval might collect sparse data. Consider changing interval length to: 4",
  5.       "suggested_value": {
  6.         "period": {
  7.           "interval": 4,
  8.           "unit": "Minutes"
  9.         }
  10.       }
  11.     }
  12.   }
  13. }

Another response might indicate that you can change filter_query (data filter) because the currently filtered data is too sparse for the model to train correctly, which can happen because the index is also ingesting data that falls outside the chosen filter. Using another filter_query can make your data more dense.

  1. {
  2.   "model": {
  3.     "filter_query": {
  4.       "message": "Data is too sparse after data filter is applied. Consider changing the data filter"
  5.     }
  6.   }
  7. }

Get detector

Introduced 1.0

Returns all information about a detector based on the detector_id.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>

Example response

  1. {
  2.   "_id": "VEHKTXwBwf_U8gjUXY2s",
  3.   "_version": 1,
  4.   "_primary_term": 1,
  5.   "_seq_no": 5,
  6.   "anomaly_detector": {
  7.     "name": "test-detector",
  8.     "description": "Test detector",
  9.     "time_field": "timestamp",
  10.     "indices": [
  11.       "server_log*"
  12.     ],
  13.     "filter_query": {
  14.       "bool": {
  15.         "filter": [
  16.           {
  17.             "range": {
  18.               "value": {
  19.                 "from": 1,
  20.                 "to": null,
  21.                 "include_lower": false,
  22.                 "include_upper": true,
  23.                 "boost": 1
  24.               }
  25.             }
  26.           }
  27.         ],
  28.         "adjust_pure_negative": true,
  29.         "boost": 1
  30.       }
  31.     },
  32.     "detection_interval": {
  33.       "period": {
  34.         "interval": 1,
  35.         "unit": "Minutes"
  36.       }
  37.     },
  38.     "window_delay": {
  39.       "period": {
  40.         "interval": 1,
  41.         "unit": "Minutes"
  42.       }
  43.     },
  44.     "shingle_size": 8,
  45.     "schema_version": 0,
  46.     "feature_attributes": [
  47.       {
  48.         "feature_id": "U0HKTXwBwf_U8gjUXY2m",
  49.         "feature_name": "test",
  50.         "feature_enabled": true,
  51.         "aggregation_query": {
  52.           "test": {
  53.             "sum": {
  54.               "field": "value"
  55.             }
  56.           }
  57.         }
  58.       }
  59.     ],
  60.     "last_update_time": 1633392680364,
  61.     "user": {
  62.       "name": "admin",
  63.       "backend_roles": [
  64.         "admin"
  65.       ],
  66.       "roles": [
  67.         "own_index",
  68.         "all_access"
  69.       ],
  70.       "custom_attribute_names": [],
  71.       "user_requested_tenant": "__user__"
  72.     },
  73.     "detector_type": "SINGLE_ENTITY"
  74.   }
  75. }

A “job” is something that you schedule to run periodically, so it’s only applicable for real-time anomaly detection and not historical analysis that you run just one time.

When you start a real-time detector, the anomaly detection plugin creates a job or if the job already exists updates it. When you start or a restart a real-time detector, the plugin creates a new real-time task that records run-time information like detector configuration snapshot, real-time job states (initializing/running/stopped), init progress, and so on.

A single detector can only have one real-time job (job ID is the same as detector ID), but it can have multiple real-time tasks because each restart of a real-time job creates a new real-time task. You can limit the number of real-time tasks with the plugins.anomaly_detection.max_old_ad_task_docs_per_detector setting.

Historical analysis doesn’t have an associated job. When you start or rerun historical analysis for a detector, the anomaly detection plugin creates a new historical batch task that tracks the historical analysis runtime information like state, coordinating/worker node, task progress, and so on. You can limit the historical task number with the plugins.anomaly_detection.max_old_ad_task_docs_per_detector setting.

Use job=true to get real-time analysis task information.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>?job=true

Example response

  1. {
  2.   "_id": "VEHKTXwBwf_U8gjUXY2s",
  3.   "_version": 1,
  4.   "_primary_term": 1,
  5.   "_seq_no": 5,
  6.   "anomaly_detector": {
  7.     "name": "test-detector",
  8.     "description": "Test detector",
  9.     "time_field": "timestamp",
  10.     "indices": [
  11.       "server_log*"
  12.     ],
  13.     "filter_query": {
  14.       "bool": {
  15.         "filter": [
  16.           {
  17.             "range": {
  18.               "value": {
  19.                 "from": 1,
  20.                 "to": null,
  21.                 "include_lower": false,
  22.                 "include_upper": true,
  23.                 "boost": 1
  24.               }
  25.             }
  26.           }
  27.         ],
  28.         "adjust_pure_negative": true,
  29.         "boost": 1
  30.       }
  31.     },
  32.     "detection_interval": {
  33.       "period": {
  34.         "interval": 1,
  35.         "unit": "Minutes"
  36.       }
  37.     },
  38.     "window_delay": {
  39.       "period": {
  40.         "interval": 1,
  41.         "unit": "Minutes"
  42.       }
  43.     },
  44.     "shingle_size": 8,
  45.     "schema_version": 0,
  46.     "feature_attributes": [
  47.       {
  48.         "feature_id": "U0HKTXwBwf_U8gjUXY2m",
  49.         "feature_name": "test",
  50.         "feature_enabled": true,
  51.         "aggregation_query": {
  52.           "test": {
  53.             "sum": {
  54.               "field": "value"
  55.             }
  56.           }
  57.         }
  58.       }
  59.     ],
  60.     "last_update_time": 1633392680364,
  61.     "user": {
  62.       "name": "admin",
  63.       "backend_roles": [
  64.         "admin"
  65.       ],
  66.       "roles": [
  67.         "own_index",
  68.         "all_access"
  69.       ],
  70.       "custom_attribute_names": [],
  71.       "user_requested_tenant": "__user__"
  72.     },
  73.     "detector_type": "SINGLE_ENTITY"
  74.   },
  75.   "anomaly_detector_job": {
  76.     "name": "VEHKTXwBwf_U8gjUXY2s",
  77.     "schedule": {
  78.       "interval": {
  79.         "start_time": 1633393656357,
  80.         "period": 1,
  81.         "unit": "Minutes"
  82.       }
  83.     },
  84.     "window_delay": {
  85.       "period": {
  86.         "interval": 1,
  87.         "unit": "Minutes"
  88.       }
  89.     },
  90.     "enabled": true,
  91.     "enabled_time": 1633393656357,
  92.     "last_update_time": 1633393656357,
  93.     "lock_duration_seconds": 60,
  94.     "user": {
  95.       "name": "admin",
  96.       "backend_roles": [
  97.         "admin"
  98.       ],
  99.       "roles": [
  100.         "own_index",
  101.         "all_access"
  102.       ],
  103.       "custom_attribute_names": [],
  104.       "user_requested_tenant": "__user__"
  105.     }
  106.   }
  107. }

Use task=true to get information for both real-time and historical analysis task information.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>?task=true

Example response

  1. {
  2.   "_id": "VEHKTXwBwf_U8gjUXY2s",
  3.   "_version": 1,
  4.   "_primary_term": 1,
  5.   "_seq_no": 5,
  6.   "anomaly_detector": {
  7.     "name": "test-detector",
  8.     "description": "Test detector",
  9.     "time_field": "timestamp",
  10.     "indices": [
  11.       "server_log*"
  12.     ],
  13.     "filter_query": {
  14.       "bool": {
  15.         "filter": [
  16.           {
  17.             "range": {
  18.               "value": {
  19.                 "from": 1,
  20.                 "to": null,
  21.                 "include_lower": false,
  22.                 "include_upper": true,
  23.                 "boost": 1
  24.               }
  25.             }
  26.           }
  27.         ],
  28.         "adjust_pure_negative": true,
  29.         "boost": 1
  30.       }
  31.     },
  32.     "detection_interval": {
  33.       "period": {
  34.         "interval": 1,
  35.         "unit": "Minutes"
  36.       }
  37.     },
  38.     "window_delay": {
  39.       "period": {
  40.         "interval": 1,
  41.         "unit": "Minutes"
  42.       }
  43.     },
  44.     "shingle_size": 8,
  45.     "schema_version": 0,
  46.     "feature_attributes": [
  47.       {
  48.         "feature_id": "U0HKTXwBwf_U8gjUXY2m",
  49.         "feature_name": "test",
  50.         "feature_enabled": true,
  51.         "aggregation_query": {
  52.           "test": {
  53.             "sum": {
  54.               "field": "value"
  55.             }
  56.           }
  57.         }
  58.       }
  59.     ],
  60.     "last_update_time": 1633392680364,
  61.     "user": {
  62.       "name": "admin",
  63.       "backend_roles": [
  64.         "admin"
  65.       ],
  66.       "roles": [
  67.         "own_index",
  68.         "all_access"
  69.       ],
  70.       "custom_attribute_names": [],
  71.       "user_requested_tenant": "__user__"
  72.     },
  73.     "detector_type": "SINGLE_ENTITY"
  74.   },
  75.   "realtime_detection_task": {
  76.     "task_id": "nkTZTXwBjd8s6RK4QlMq",
  77.     "last_update_time": 1633393776375,
  78.     "started_by": "admin",
  79.     "error": "",
  80.     "state": "RUNNING",
  81.     "detector_id": "VEHKTXwBwf_U8gjUXY2s",
  82.     "task_progress": 0,
  83.     "init_progress": 1,
  84.     "execution_start_time": 1633393656362,
  85.     "is_latest": true,
  86.     "task_type": "REALTIME_SINGLE_ENTITY",
  87.     "coordinating_node": "SWD7ihu9TaaW1zKwFZNVNg",
  88.     "detector": {
  89.       "name": "test-detector",
  90.       "description": "Test detector",
  91.       "time_field": "timestamp",
  92.       "indices": [
  93.         "server_log*"
  94.       ],
  95.       "filter_query": {
  96.         "bool": {
  97.           "filter": [
  98.             {
  99.               "range": {
  100.                 "value": {
  101.                   "from": 1,
  102.                   "to": null,
  103.                   "include_lower": false,
  104.                   "include_upper": true,
  105.                   "boost": 1
  106.                 }
  107.               }
  108.             }
  109.           ],
  110.           "adjust_pure_negative": true,
  111.           "boost": 1
  112.         }
  113.       },
  114.       "detection_interval": {
  115.         "period": {
  116.           "interval": 1,
  117.           "unit": "Minutes"
  118.         }
  119.       },
  120.       "window_delay": {
  121.         "period": {
  122.           "interval": 1,
  123.           "unit": "Minutes"
  124.         }
  125.       },
  126.       "shingle_size": 8,
  127.       "schema_version": 0,
  128.       "feature_attributes": [
  129.         {
  130.           "feature_id": "U0HKTXwBwf_U8gjUXY2m",
  131.           "feature_name": "test",
  132.           "feature_enabled": true,
  133.           "aggregation_query": {
  134.             "test": {
  135.               "sum": {
  136.                 "field": "value"
  137.               }
  138.             }
  139.           }
  140.         }
  141.       ],
  142.       "last_update_time": 1633392680364,
  143.       "user": {
  144.         "name": "admin",
  145.         "backend_roles": [
  146.           "admin"
  147.         ],
  148.         "roles": [
  149.           "own_index",
  150.           "all_access"
  151.         ],
  152.         "custom_attribute_names": [],
  153.         "user_requested_tenant": "__user__"
  154.       },
  155.       "detector_type": "SINGLE_ENTITY"
  156.     },
  157.     "estimated_minutes_left": 0,
  158.     "user": {
  159.       "name": "admin",
  160.       "backend_roles": [
  161.         "admin"
  162.       ],
  163.       "roles": [
  164.         "own_index",
  165.         "all_access"
  166.       ],
  167.       "custom_attribute_names": [],
  168.       "user_requested_tenant": "__user__"
  169.     }
  170.   },
  171.   "historical_analysis_task": {
  172.     "task_id": "99DaTXwB6HknB84StRN1",
  173.     "last_update_time": 1633393797040,
  174.     "started_by": "admin",
  175.     "state": "RUNNING",
  176.     "detector_id": "VEHKTXwBwf_U8gjUXY2s",
  177.     "task_progress": 0.89285713,
  178.     "init_progress": 1,
  179.     "current_piece": 1633328940000,
  180.     "execution_start_time": 1633393751412,
  181.     "is_latest": true,
  182.     "task_type": "HISTORICAL_SINGLE_ENTITY",
  183.     "coordinating_node": "SWD7ihu9TaaW1zKwFZNVNg",
  184.     "worker_node": "2Z4q22BySEyzakYt_A0A2A",
  185.     "detector": {
  186.       "name": "test-detector",
  187.       "description": "Test detector",
  188.       "time_field": "timestamp",
  189.       "indices": [
  190.         "server_log*"
  191.       ],
  192.       "filter_query": {
  193.         "bool": {
  194.           "filter": [
  195.             {
  196.               "range": {
  197.                 "value": {
  198.                   "from": 1,
  199.                   "to": null,
  200.                   "include_lower": false,
  201.                   "include_upper": true,
  202.                   "boost": 1
  203.                 }
  204.               }
  205.             }
  206.           ],
  207.           "adjust_pure_negative": true,
  208.           "boost": 1
  209.         }
  210.       },
  211.       "detection_interval": {
  212.         "period": {
  213.           "interval": 1,
  214.           "unit": "Minutes"
  215.         }
  216.       },
  217.       "window_delay": {
  218.         "period": {
  219.           "interval": 1,
  220.           "unit": "Minutes"
  221.         }
  222.       },
  223.       "shingle_size": 8,
  224.       "schema_version": 0,
  225.       "feature_attributes": [
  226.         {
  227.           "feature_id": "U0HKTXwBwf_U8gjUXY2m",
  228.           "feature_name": "test",
  229.           "feature_enabled": true,
  230.           "aggregation_query": {
  231.             "test": {
  232.               "sum": {
  233.                 "field": "value"
  234.               }
  235.             }
  236.           }
  237.         }
  238.       ],
  239.       "last_update_time": 1633392680364,
  240.       "user": {
  241.         "name": "admin",
  242.         "backend_roles": [
  243.           "admin"
  244.         ],
  245.         "roles": [
  246.           "own_index",
  247.           "all_access"
  248.         ],
  249.         "custom_attribute_names": [],
  250.         "user_requested_tenant": "__user__"
  251.       },
  252.       "detector_type": "SINGLE_ENTITY"
  253.     },
  254.     "detection_date_range": {
  255.       "start_time": 1632788951329,
  256.       "end_time": 1633393751329
  257.     },
  258.     "user": {
  259.       "name": "admin",
  260.       "backend_roles": [
  261.         "admin"
  262.       ],
  263.       "roles": [
  264.         "own_index",
  265.         "all_access"
  266.       ],
  267.       "custom_attribute_names": [],
  268.       "user_requested_tenant": "__user__"
  269.     }
  270.   }
  271. }

Update detector

Introduced 1.0

Updates a detector with any changes, including the description or adding or removing of features. To update a detector, you need to first stop both real-time detection and historical analysis.

You can’t update a category field.

Request

  1. PUT _plugins/_anomaly_detection/detectors/<detectorId>
  2. {
  3.   "name": "test-detector",
  4.   "description": "Test update detector",
  5.   "time_field": "timestamp",
  6.   "indices": [
  7.     "server_log*"
  8.   ],
  9.   "feature_attributes": [
  10.     {
  11.       "feature_name": "test",
  12.       "feature_enabled": true,
  13.       "aggregation_query": {
  14.         "test": {
  15.           "sum": {
  16.             "field": "value"
  17.           }
  18.         }
  19.       }
  20.     }
  21.   ],
  22.   "filter_query": {
  23.     "bool": {
  24.       "filter": [
  25.         {
  26.           "range": {
  27.             "value": {
  28.               "gt": 1
  29.             }
  30.           }
  31.         }
  32.       ],
  33.       "adjust_pure_negative": true,
  34.       "boost": 1
  35.     }
  36.   },
  37.   "detection_interval": {
  38.     "period": {
  39.       "interval": 1,
  40.       "unit": "Minutes"
  41.     }
  42.   },
  43.   "window_delay": {
  44.     "period": {
  45.       "interval": 1,
  46.       "unit": "Minutes"
  47.     }
  48.   }
  49. }

Example response

  1. {
  2.   "_id": "VEHKTXwBwf_U8gjUXY2s",
  3.   "_version": 2,
  4.   "_seq_no": 7,
  5.   "anomaly_detector": {
  6.     "name": "test-detector",
  7.     "description": "Test update detector",
  8.     "time_field": "timestamp",
  9.     "indices": [
  10.       "server_log*"
  11.     ],
  12.     "filter_query": {
  13.       "bool": {
  14.         "filter": [
  15.           {
  16.             "range": {
  17.               "value": {
  18.                 "from": 1,
  19.                 "to": null,
  20.                 "include_lower": false,
  21.                 "include_upper": true,
  22.                 "boost": 1
  23.               }
  24.             }
  25.           }
  26.         ],
  27.         "adjust_pure_negative": true,
  28.         "boost": 1
  29.       }
  30.     },
  31.     "detection_interval": {
  32.       "period": {
  33.         "interval": 1,
  34.         "unit": "Minutes"
  35.       }
  36.     },
  37.     "window_delay": {
  38.       "period": {
  39.         "interval": 1,
  40.         "unit": "Minutes"
  41.       }
  42.     },
  43.     "shingle_size": 8,
  44.     "schema_version": 0,
  45.     "feature_attributes": [
  46.       {
  47.         "feature_id": "3kHiTXwBwf_U8gjUlY15",
  48.         "feature_name": "test",
  49.         "feature_enabled": true,
  50.         "aggregation_query": {
  51.           "test": {
  52.             "sum": {
  53.               "field": "value"
  54.             }
  55.           }
  56.         }
  57.       }
  58.     ],
  59.     "last_update_time": 1633394267522,
  60.     "user": {
  61.       "name": "admin",
  62.       "backend_roles": [
  63.         "admin"
  64.       ],
  65.       "roles": [
  66.         "own_index",
  67.         "all_access"
  68.       ],
  69.       "custom_attribute_names": [],
  70.       "user_requested_tenant": "__user__"
  71.     },
  72.     "detector_type": "SINGLE_ENTITY"
  73.   },
  74.   "_primary_term": 1
  75. }

Delete detector

Introduced 1.0

Deletes a detector based on the detector_id. To delete a detector, you need to first stop both real-time detection and historical analysis.

Request

  1. DELETE _plugins/_anomaly_detection/detectors/<detectorId>

Example response

  1. {
  2.   "_index": ".opensearch-anomaly-detectors",
  3.   "_id": "70TxTXwBjd8s6RK4j1Pj",
  4.   "_version": 2,
  5.   "result": "deleted",
  6.   "forced_refresh": true,
  7.   "_shards": {
  8.     "total": 2,
  9.     "successful": 2,
  10.     "failed": 0
  11.   },
  12.   "_seq_no": 9,
  13.   "_primary_term": 1
  14. }

Preview detector

Introduced 1.0

Passes a date range to the anomaly detector to return any anomalies within that date range.

To preview a single-entity detector:

Request

  1. POST _plugins/_anomaly_detection/detectors/_preview
  2. {
  3.   "period_start": 1633048868000,
  4.   "period_end": 1633394468000,
  5.   "detector": {
  6.     "name": "test-detector",
  7.     "description": "Test update detector",
  8.     "time_field": "timestamp",
  9.     "indices": [
  10.       "server_log*"
  11.     ],
  12.     "feature_attributes": [
  13.       {
  14.         "feature_name": "test",
  15.         "feature_enabled": true,
  16.         "aggregation_query": {
  17.           "test": {
  18.             "sum": {
  19.               "field": "value"
  20.             }
  21.           }
  22.         }
  23.       }
  24.     ],
  25.     "filter_query": {
  26.       "bool": {
  27.         "filter": [
  28.           {
  29.             "range": {
  30.               "value": {
  31.                 "gt": 1
  32.               }
  33.             }
  34.           }
  35.         ],
  36.         "adjust_pure_negative": true,
  37.         "boost": 1
  38.       }
  39.     },
  40.     "detection_interval": {
  41.       "period": {
  42.         "interval": 1,
  43.         "unit": "Minutes"
  44.       }
  45.     },
  46.     "window_delay": {
  47.       "period": {
  48.         "interval": 1,
  49.         "unit": "Minutes"
  50.       }
  51.     }
  52.   }
  53. }

Example response

  1. {
  2.   "anomaly_result": [
  3.     {
  4.       "detector_id": null,
  5.       "data_start_time": 1633049280000,
  6.       "data_end_time": 1633049340000,
  7.       "schema_version": 0,
  8.       "feature_data": [
  9.         {
  10.           "feature_id": "8EHmTXwBwf_U8gjU0Y0u",
  11.           "feature_name": "test",
  12.           "data": 0
  13.         }
  14.       ],
  15.       "anomaly_grade": 0,
  16.       "confidence": 0
  17.     },
  18.     ...
  19.   ],
  20.   "anomaly_detector": {
  21.     "name": "test-detector",
  22.     "description": "Test update detector",
  23.     "time_field": "timestamp",
  24.     "indices": [
  25.       "server_log*"
  26.     ],
  27.     "filter_query": {
  28.       "bool": {
  29.         "filter": [
  30.           {
  31.             "range": {
  32.               "value": {
  33.                 "from": 1,
  34.                 "to": null,
  35.                 "include_lower": false,
  36.                 "include_upper": true,
  37.                 "boost": 1
  38.               }
  39.             }
  40.           }
  41.         ],
  42.         "adjust_pure_negative": true,
  43.         "boost": 1
  44.       }
  45.     },
  46.     "detection_interval": {
  47.       "period": {
  48.         "interval": 1,
  49.         "unit": "Minutes"
  50.       }
  51.     },
  52.     "window_delay": {
  53.       "period": {
  54.         "interval": 1,
  55.         "unit": "Minutes"
  56.       }
  57.     },
  58.     "shingle_size": 8,
  59.     "schema_version": 0,
  60.     "feature_attributes": [
  61.       {
  62.         "feature_id": "8EHmTXwBwf_U8gjU0Y0u",
  63.         "feature_name": "test",
  64.         "feature_enabled": true,
  65.         "aggregation_query": {
  66.           "test": {
  67.             "sum": {
  68.               "field": "value"
  69.             }
  70.           }
  71.         }
  72.       }
  73.     ],
  74.     "detector_type": "SINGLE_ENTITY"
  75.   }
  76. }

If you specify a category field, each result is associated with an entity:

Request

  1. POST _plugins/_anomaly_detection/detectors/_preview
  2. {
  3.   "period_start": 1633048868000,
  4.   "period_end": 1633394468000,
  5.   "detector": {
  6.     "name": "test-detector",
  7.     "description": "Test update detector",
  8.     "time_field": "timestamp",
  9.     "indices": [
  10.       "server_log*"
  11.     ],
  12.     "feature_attributes": [
  13.       {
  14.         "feature_name": "test",
  15.         "feature_enabled": true,
  16.         "aggregation_query": {
  17.           "test": {
  18.             "sum": {
  19.               "field": "value"
  20.             }
  21.           }
  22.         }
  23.       }
  24.     ],
  25.     "filter_query": {
  26.       "bool": {
  27.         "filter": [
  28.           {
  29.             "range": {
  30.               "value": {
  31.                 "gt": 1
  32.               }
  33.             }
  34.           }
  35.         ],
  36.         "adjust_pure_negative": true,
  37.         "boost": 1
  38.       }
  39.     },
  40.     "detection_interval": {
  41.       "period": {
  42.         "interval": 1,
  43.         "unit": "Minutes"
  44.       }
  45.     },
  46.     "window_delay": {
  47.       "period": {
  48.         "interval": 1,
  49.         "unit": "Minutes"
  50.       }
  51.     },
  52.     "category_field": [
  53.       "error_type"
  54.     ]
  55.   }
  56. }

Example response

  1. {
  2.   "anomaly_result": [
  3.     {
  4.       "detector_id": null,
  5.       "data_start_time": 1633049280000,
  6.       "data_end_time": 1633049340000,
  7.       "schema_version": 0,
  8.       "feature_data": [
  9.         {
  10.           "feature_id": "tkTpTXwBjd8s6RK4DlOZ",
  11.           "feature_name": "test",
  12.           "data": 0
  13.         }
  14.       ],
  15.       "anomaly_grade": 0,
  16.       "confidence": 0,
  17.       "entity": [
  18.         {
  19.           "name": "error_type",
  20.           "value": "error1"
  21.         }
  22.       ]
  23.     },
  24.     ...
  25.   ],
  26.   "anomaly_detector": {
  27.     "name": "test-detector",
  28.     "description": "Test update detector",
  29.     "time_field": "timestamp",
  30.     "indices": [
  31.       "server_log*"
  32.     ],
  33.     "filter_query": {
  34.       "bool": {
  35.         "filter": [
  36.           {
  37.             "range": {
  38.               "value": {
  39.                 "from": 1,
  40.                 "to": null,
  41.                 "include_lower": false,
  42.                 "include_upper": true,
  43.                 "boost": 1
  44.               }
  45.             }
  46.           }
  47.         ],
  48.         "adjust_pure_negative": true,
  49.         "boost": 1
  50.       }
  51.     },
  52.     "detection_interval": {
  53.       "period": {
  54.         "interval": 1,
  55.         "unit": "Minutes"
  56.       }
  57.     },
  58.     "window_delay": {
  59.       "period": {
  60.         "interval": 1,
  61.         "unit": "Minutes"
  62.       }
  63.     },
  64.     "shingle_size": 8,
  65.     "schema_version": 0,
  66.     "feature_attributes": [
  67.       {
  68.         "feature_id": "tkTpTXwBjd8s6RK4DlOZ",
  69.         "feature_name": "test",
  70.         "feature_enabled": true,
  71.         "aggregation_query": {
  72.           "test": {
  73.             "sum": {
  74.               "field": "value"
  75.             }
  76.           }
  77.         }
  78.       }
  79.     ],
  80.     "category_field": [
  81.       "error_type"
  82.     ],
  83.     "detector_type": "MULTI_ENTITY"
  84.   }
  85. }

You can preview a detector with the detector ID:

  1. POST _plugins/_anomaly_detection/detectors/_preview
  2. {
  3.   "detector_id": "VEHKTXwBwf_U8gjUXY2s",
  4.   "period_start": 1633048868000,
  5.   "period_end": 1633394468000
  6. }

Or:

  1. POST _opendistro/_anomaly_detection/detectors/VEHKTXwBwf_U8gjUXY2s/_preview
  2. {
  3.   "period_start": 1633048868000,
  4.   "period_end": 1633394468000
  5. }

Example response

  1. {
  2.   "anomaly_result": [
  3.     {
  4.       "detector_id": "VEHKTXwBwf_U8gjUXY2s",
  5.       "data_start_time": 1633049280000,
  6.       "data_end_time": 1633049340000,
  7.       "schema_version": 0,
  8.       "feature_data": [
  9.         {
  10.           "feature_id": "3kHiTXwBwf_U8gjUlY15",
  11.           "feature_name": "test",
  12.           "data": 0
  13.         }
  14.       ],
  15.       "anomaly_grade": 0,
  16.       "confidence": 0,
  17.       "user": {
  18.         "name": "admin",
  19.         "backend_roles": [
  20.           "admin"
  21.         ],
  22.         "roles": [
  23.           "own_index",
  24.           "all_access"
  25.         ],
  26.         "custom_attribute_names": [],
  27.         "user_requested_tenant": "__user__"
  28.       }
  29.     },
  30.     ...
  31.   ],
  32.   "anomaly_detector": {
  33.     "name": "test-detector",
  34.     "description": "Test update detector",
  35.     "time_field": "timestamp",
  36.     "indices": [
  37.       "server_log*"
  38.     ],
  39.     "filter_query": {
  40.       "bool": {
  41.         "filter": [
  42.           {
  43.             "range": {
  44.               "value": {
  45.                 "from": 1,
  46.                 "to": null,
  47.                 "include_lower": false,
  48.                 "include_upper": true,
  49.                 "boost": 1
  50.               }
  51.             }
  52.           }
  53.         ],
  54.         "adjust_pure_negative": true,
  55.         "boost": 1
  56.       }
  57.     },
  58.     "detection_interval": {
  59.       "period": {
  60.         "interval": 1,
  61.         "unit": "Minutes"
  62.       }
  63.     },
  64.     "window_delay": {
  65.       "period": {
  66.         "interval": 1,
  67.         "unit": "Minutes"
  68.       }
  69.     },
  70.     "shingle_size": 8,
  71.     "schema_version": 0,
  72.     "feature_attributes": [
  73.       {
  74.         "feature_id": "3kHiTXwBwf_U8gjUlY15",
  75.         "feature_name": "test",
  76.         "feature_enabled": true,
  77.         "aggregation_query": {
  78.           "test": {
  79.             "sum": {
  80.               "field": "value"
  81.             }
  82.           }
  83.         }
  84.       }
  85.     ],
  86.     "last_update_time": 1633394267522,
  87.     "user": {
  88.       "name": "admin",
  89.       "backend_roles": [
  90.         "admin"
  91.       ],
  92.       "roles": [
  93.         "own_index",
  94.         "all_access"
  95.       ],
  96.       "custom_attribute_names": [],
  97.       "user_requested_tenant": "__user__"
  98.     },
  99.     "detector_type": "SINGLE_ENTITY"
  100.   }
  101. }

Start detector job

Introduced 1.0

Starts a real-time or historical anomaly detector job.

To start a real-time detector job:

Request

  1. POST _plugins/_anomaly_detection/detectors/<detectorId>/_start

Example response

  1. {
  2.   "_id": "VEHKTXwBwf_U8gjUXY2s",
  3.   "_version": 3,
  4.   "_seq_no": 6,
  5.   "_primary_term": 1
  6. }

The _id represents the real-time job ID, which is the same as the detector ID.

To start historical analysis:

  1. POST _plugins/_anomaly_detection/detectors/<detectorId>/_start
  2. {
  3.   "start_time": 1633048868000,
  4.   "end_time": 1633394468000
  5. }

Example response

  1. {
  2.   "_id": "f9DsTXwB6HknB84SoRTY",
  3.   "_version": 1,
  4.   "_seq_no": 958,
  5.   "_primary_term": 1
  6. }

The _id represents the historical batch task ID, which is a random universally unique identifier (UUID).

Stop detector job

Introduced 1.0

Stops a real-time or historical anomaly detector job.

To stop a real-time detector job:

Request

  1. POST _plugins/_anomaly_detection/detectors/<detectorId>/_stop

Example response

  1. {
  2.   "_id": "VEHKTXwBwf_U8gjUXY2s",
  3.   "_version": 0,
  4.   "_seq_no": 0,
  5.   "_primary_term": 0
  6. }

To stop historical analysis:

Introduced 1.1

  1. POST _plugins/_anomaly_detection/detectors/<detectorId>/_stop?historical=true

Example response

  1. {
  2.   "_id": "f9DsTXwB6HknB84SoRTY",
  3.   "_version": 0,
  4.   "_seq_no": 0,
  5.   "_primary_term": 0
  6. }

Search detector

Introduced 1.0

Returns all anomaly detectors for a search query.

To search detectors using the server_log* index:

Request

  1. GET _plugins/_anomaly_detection/detectors/_search
  2. POST _plugins/_anomaly_detection/detectors/_search
  3. {
  4.   "query": {
  5.     "wildcard": {
  6.       "indices": {
  7.         "value": "server_log*"
  8.       }
  9.     }
  10.   }
  11. }

Example response

  1. {
  2.   "took": 2,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 1,
  6.     "successful": 1,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 4,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": 1,
  16.     "hits": [
  17.       {
  18.         "_index": ".opensearch-anomaly-detectors",
  19.         "_id": "Zi5zTXwBwf_U8gjUTfJG",
  20.         "_version": 1,
  21.         "_seq_no": 1,
  22.         "_primary_term": 1,
  23.         "_score": 1,
  24.         "_source": {
  25.           "name": "test",
  26.           "description": "test",
  27.           "time_field": "timestamp",
  28.           "indices": [
  29.             "server_log"
  30.           ],
  31.           "filter_query": {
  32.             "match_all": {
  33.               "boost": 1
  34.             }
  35.           },
  36.           "detection_interval": {
  37.             "period": {
  38.               "interval": 5,
  39.               "unit": "Minutes"
  40.             }
  41.           },
  42.           "window_delay": {
  43.             "period": {
  44.               "interval": 1,
  45.               "unit": "Minutes"
  46.             }
  47.           },
  48.           "shingle_size": 8,
  49.           "schema_version": 0,
  50.           "feature_attributes": [
  51.             {
  52.               "feature_id": "ZS5zTXwBwf_U8gjUTfIn",
  53.               "feature_name": "test_feature",
  54.               "feature_enabled": true,
  55.               "aggregation_query": {
  56.                 "test_feature": {
  57.                   "sum": {
  58.                     "field": "value"
  59.                   }
  60.                 }
  61.               }
  62.             }
  63.           ],
  64.           "last_update_time": 1633386974533,
  65.           "category_field": [
  66.             "error_type"
  67.           ],
  68.           "user": {
  69.             "name": "admin",
  70.             "backend_roles": [
  71.               "admin"
  72.             ],
  73.             "roles": [
  74.               "own_index",
  75.               "all_access"
  76.             ],
  77.             "custom_attribute_names": [],
  78.             "user_requested_tenant": "__user__"
  79.           },
  80.           "detector_type": "MULTI_ENTITY"
  81.         }
  82.       },
  83.       ...
  84.     ]
  85.   }
  86. }

Search detector tasks

Introduced 1.1

Searches detector tasks.

To search for the latest detector level historical analysis task for a high cardinality detector

Request

  1. GET _plugins/_anomaly_detection/detectors/tasks/_search
  2. POST _plugins/_anomaly_detection/detectors/tasks/_search
  3. {
  4.   "query": {
  5.     "bool": {
  6.       "filter": [
  7.         {
  8.           "term": {
  9.             "detector_id": "Zi5zTXwBwf_U8gjUTfJG"
  10.           }
  11.         },
  12.         {
  13.           "term": {
  14.             "task_type": "HISTORICAL_HC_DETECTOR"
  15.           }
  16.         },
  17.         {
  18.           "term": {
  19.             "is_latest": "true"
  20.           }
  21.         }
  22.       ]
  23.     }
  24.   }
  25. }

Example response

  1. {
  2.   "took": 1,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 1,
  6.     "successful": 1,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 1,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": 0,
  16.     "hits": [
  17.       {
  18.         "_index": ".opensearch-anomaly-detection-state",
  19.         "_id": "fm-RTXwBYwCbWecgB753",
  20.         "_version": 34,
  21.         "_seq_no": 928,
  22.         "_primary_term": 1,
  23.         "_score": 0,
  24.         "_source": {
  25.           "detector_id": "Zi5zTXwBwf_U8gjUTfJG",
  26.           "error": "",
  27.           "detection_date_range": {
  28.             "start_time": 1630794960000,
  29.             "end_time": 1633386960000
  30.           },
  31.           "task_progress": 1,
  32.           "last_update_time": 1633389090738,
  33.           "execution_start_time": 1633388922742,
  34.           "state": "FINISHED",
  35.           "coordinating_node": "2Z4q22BySEyzakYt_A0A2A",
  36.           "task_type": "HISTORICAL_HC_DETECTOR",
  37.           "execution_end_time": 1633389090738,
  38.           "started_by": "admin",
  39.           "init_progress": 0,
  40.           "is_latest": true,
  41.           "detector": {
  42.             "category_field": [
  43.               "error_type"
  44.             ],
  45.             "description": "test",
  46.             "ui_metadata": {
  47.               "features": {
  48.                 "test_feature": {
  49.                   "aggregationBy": "sum",
  50.                   "aggregationOf": "value",
  51.                   "featureType": "simple_aggs"
  52.                 }
  53.               },
  54.               "filters": []
  55.             },
  56.             "feature_attributes": [
  57.               {
  58.                 "feature_id": "ZS5zTXwBwf_U8gjUTfIn",
  59.                 "feature_enabled": true,
  60.                 "feature_name": "test_feature",
  61.                 "aggregation_query": {
  62.                   "test_feature": {
  63.                     "sum": {
  64.                       "field": "value"
  65.                     }
  66.                   }
  67.                 }
  68.               }
  69.             ],
  70.             "schema_version": 0,
  71.             "time_field": "timestamp",
  72.             "last_update_time": 1633386974533,
  73.             "indices": [
  74.               "server_log"
  75.             ],
  76.             "window_delay": {
  77.               "period": {
  78.                 "unit": "Minutes",
  79.                 "interval": 1
  80.               }
  81.             },
  82.             "detection_interval": {
  83.               "period": {
  84.                 "unit": "Minutes",
  85.                 "interval": 5
  86.               }
  87.             },
  88.             "name": "testhc",
  89.             "filter_query": {
  90.               "match_all": {
  91.                 "boost": 1
  92.               }
  93.             },
  94.             "shingle_size": 8,
  95.             "user": {
  96.               "backend_roles": [
  97.                 "admin"
  98.               ],
  99.               "custom_attribute_names": [],
  100.               "roles": [
  101.                 "own_index",
  102.                 "all_access"
  103.               ],
  104.               "name": "admin",
  105.               "user_requested_tenant": "__user__"
  106.             },
  107.             "detector_type": "MULTI_ENTITY"
  108.           },
  109.           "user": {
  110.             "backend_roles": [
  111.               "admin"
  112.             ],
  113.             "custom_attribute_names": [],
  114.             "roles": [
  115.               "own_index",
  116.               "all_access"
  117.             ],
  118.             "name": "admin",
  119.             "user_requested_tenant": "__user__"
  120.           }
  121.         }
  122.       }
  123.     ]
  124.   }
  125. }

To search for the latest entity-level tasks for the historical analysis of a high cardinality detector:

Request

  1. GET _plugins/_anomaly_detection/detectors/tasks/_search
  2. POST _plugins/_anomaly_detection/detectors/tasks/_search
  3. {
  4.   "query": {
  5.     "bool": {
  6.       "filter": [
  7.         {
  8.           "term": {
  9.             "detector_id": "Zi5zTXwBwf_U8gjUTfJG"
  10.           }
  11.         },
  12.         {
  13.           "term": {
  14.             "task_type": "HISTORICAL_HC_ENTITY"
  15.           }
  16.         },
  17.         {
  18.           "term": {
  19.             "is_latest": "true"
  20.           }
  21.         }
  22.       ]
  23.     }
  24.   },
  25.   "sort": [
  26.     {
  27.       "execution_start_time": {
  28.         "order": "desc"
  29.       }
  30.     }
  31.   ],
  32.   "size": 100
  33. }

To search and aggregate states for all entity-level historical tasks:

The parent_task_id is the same as the task ID that you can get with the profile detector API: GET _plugins/_anomaly_detection/detectors/<detector_ID>/_profile/ad_task.

Request

  1. GET _plugins/_anomaly_detection/detectors/tasks/_search
  2. POST _plugins/_anomaly_detection/detectors/tasks/_search
  3. {
  4.   "size": 0,
  5.   "query": {
  6.     "bool": {
  7.       "filter": [
  8.         {
  9.           "term": {
  10.             "detector_id": {
  11.               "value": "Zi5zTXwBwf_U8gjUTfJG",
  12.               "boost": 1
  13.             }
  14.           }
  15.         },
  16.         {
  17.           "term": {
  18.             "parent_task_id": {
  19.               "value": "fm-RTXwBYwCbWecgB753",
  20.               "boost": 1
  21.             }
  22.           }
  23.         },
  24.         {
  25.           "terms": {
  26.             "task_type": [
  27.               "HISTORICAL_HC_ENTITY"
  28.             ],
  29.             "boost": 1
  30.           }
  31.         }
  32.       ]
  33.     }
  34.   },
  35.   "aggs": {
  36.     "test": {
  37.       "terms": {
  38.         "field": "state",
  39.         "size": 100
  40.       }
  41.     }
  42.   }
  43. }

Example response

  1. {
  2.   "took": 2,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 1,
  6.     "successful": 1,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 32,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": null,
  16.     "hits": []
  17.   },
  18.   "aggregations": {
  19.     "test": {
  20.       "doc_count_error_upper_bound": 0,
  21.       "sum_other_doc_count": 0,
  22.       "buckets": [
  23.         {
  24.           "key": "FINISHED",
  25.           "doc_count": 32
  26.         }
  27.       ]
  28.     }
  29.   }
  30. }

Search detector result

Introduced 1.0

Returns all results for a search query.

You have the following search options:

  • To search only the default result index, simply use the search API:

    1. POST _plugins/_anomaly_detection/detectors/results/_search/

  • To search both the custom result index and default result index, you can either add the custom result index to the search API:

    1. POST _plugins/_anomaly_detection/detectors/results/_search/<custom_result_index>

    Or, add the custom result index and set the only_query_custom_result_index parameter to false:

    1. POST _plugins/_anomaly_detection/detectors/results/_search/<custom_result_index>?only_query_custom_result_index=false

  • To search only the custom result index, add the custom result index to the search API and set the only_query_custom_result_index parameter to true:

    1. POST _plugins/_anomaly_detection/detectors/results/_search/<custom_result_index>?only_query_custom_result_index=true

The following example searches anomaly results for grade greater than 0 for real-time analysis:

Request

  1. GET _plugins/_anomaly_detection/detectors/results/_search/opensearch-ad-plugin-result-test
  2. POST _plugins/_anomaly_detection/detectors/results/_search/opensearch-ad-plugin-result-test
  3. {
  4.   "query": {
  5.     "bool": {
  6.       "filter": [
  7.         {
  8.           "term": {
  9.             "detector_id": "EWy02nwBm38sXcF2AiFJ"
  10.           }
  11.         },
  12.         {
  13.           "range": {
  14.             "anomaly_grade": {
  15.               "gt": 0
  16.             }
  17.           }
  18.         }
  19.       ],
  20.       "must_not": [
  21.         {
  22.           "exists": {
  23.             "field": "task_id"
  24.           }
  25.         }
  26.       ]
  27.     }
  28.   }
  29. }

If you specify the custom result index like in this example, the search results API searches both the default result indices and custom result indices.

If you don’t specify the custom result index and you just use the _plugins/_anomaly_detection/detectors/results/_search URL, the anomaly detection plugin searches only the default result indices.

Real-time detection doesn’t persist the task ID in the anomaly result, so the task ID will be null.

For information about the response body fields, see Anomaly result mapping.

Example response

  1. {
  2.   "took": 4,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 3,
  6.     "successful": 3,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 90,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": 0,
  16.     "hits": [
  17.       {
  18.         "_index": ".opensearch-anomaly-results-history-2021.10.04-1",
  19.         "_id": "686KTXwB6HknB84SMr6G",
  20.         "_version": 1,
  21.         "_seq_no": 103622,
  22.         "_primary_term": 1,
  23.         "_score": 0,
  24.         "_source": {
  25.           "detector_id": "EWy02nwBm38sXcF2AiFJ",
  26.           "confidence": 0.918886275269358,
  27.           "model_id": "EWy02nwBm38sXcF2AiFJ_entity_error16",
  28.           "schema_version": 4,
  29.           "anomaly_score": 1.1093755891885446,
  30.           "execution_start_time": 1633388475001,
  31.           "data_end_time": 1633388414989,
  32.           "data_start_time": 1633388114989,
  33.           "feature_data": [
  34.             {
  35.               "feature_id": "ZS5zTXwBwf_U8gjUTfIn",
  36.               "feature_name": "test_feature",
  37.               "data": 0.532
  38.             }
  39.           ],
  40.           "relevant_attribution": [
  41.             {
  42.               "feature_id": "ZS5zTXwBwf_U8gjUTfIn",
  43.               "data": 1.0
  44.             }
  45.           ],
  46.           "expected_values": [
  47.             {
  48.               "likelihood": 1,
  49.               "value_list": [
  50.                 {
  51.                   "feature_id": "ZS5zTXwBwf_U8gjUTfIn",
  52.                   "data": 2
  53.                 }
  54.               ]
  55.             }
  56.           ],
  57.           "execution_end_time": 1633388475014,
  58.           "user": {
  59.             "backend_roles": [
  60.               "admin"
  61.             ],
  62.             "custom_attribute_names": [],
  63.             "roles": [
  64.               "own_index",
  65.               "all_access"
  66.             ],
  67.             "name": "admin",
  68.             "user_requested_tenant": "__user__"
  69.           },
  70.           "anomaly_grade": 0.031023547546561225,
  71.           "entity": [
  72.             {
  73.               "name": "error_type",
  74.               "value": "error16"
  75.             }
  76.           ]
  77.         }
  78.       },
  79.       ...
  80.     ]
  81.   }
  82. }

You can run historical analysis as many times as you like. So, multiple tasks might exist for the same detector.

You can search for the latest historical batch task first and then search the historical batch task results.

To search anomaly results for grade greater than 0 for historical analysis with the task_id:

Request

  1. GET _plugins/_anomaly_detection/detectors/results/_search
  2. POST _plugins/_anomaly_detection/detectors/results/_search
  3. {
  4.   "query": {
  5.     "bool": {
  6.       "filter": [
  7.         {
  8.           "term": {
  9.             "detector_id": "Zi5zTXwBwf_U8gjUTfJG"
  10.           }
  11.         },
  12.         {
  13.           "range": {
  14.             "anomaly_grade": {
  15.               "gt": 0
  16.             }
  17.           }
  18.         },
  19.         {
  20.           "term": {
  21.             "task_id": "fm-RTXwBYwCbWecgB753"
  22.           }
  23.         }
  24.       ]
  25.     }
  26.   }
  27. }

Example response

  1. {
  2.   "took": 915,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 3,
  6.     "successful": 3,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 4115,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": 0,
  16.     "hits": [
  17.       {
  18.         "_index": ".opensearch-anomaly-results-history-2021.10.04-1",
  19.         "_id": "VRyRTXwBDx7vzPBV8jYC",
  20.         "_version": 1,
  21.         "_seq_no": 149657,
  22.         "_primary_term": 1,
  23.         "_score": 0,
  24.         "_source": {
  25.           "detector_id": "Zi5zTXwBwf_U8gjUTfJG",
  26.           "confidence": 0.9642989263957601,
  27.           "task_id": "fm-RTXwBYwCbWecgB753",
  28.           "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error24",
  29.           "schema_version": 4,
  30.           "anomaly_score": 1.2260712437521946,
  31.           "execution_start_time": 1633388982692,
  32.           "data_end_time": 1631721300000,
  33.           "data_start_time": 1631721000000,
  34.           "feature_data": [
  35.             {
  36.               "feature_id": "ZS5zTXwBwf_U8gjUTfIn",
  37.               "feature_name": "test_feature",
  38.               "data": 10
  39.             }
  40.           ],
  41.           "execution_end_time": 1633388982709,
  42.           "user": {
  43.             "backend_roles": [
  44.               "admin"
  45.             ],
  46.             "custom_attribute_names": [],
  47.             "roles": [
  48.               "own_index",
  49.               "all_access"
  50.             ],
  51.             "name": "admin",
  52.             "user_requested_tenant": "__user__"
  53.           },
  54.           "anomaly_grade": 0.14249628345655782,
  55.           "entity": [
  56.             {
  57.               "name": "error_type",
  58.               "value": "error1"
  59.             }
  60.           ]
  61.         }
  62.       },
  63.       ...
  64.     ]
  65.   }
  66. }

Search top anomalies

Introduced 1.2

Returns the top anomaly results for a high-cardinality detector, bucketed by categorical field values.

You can pass a historical boolean parameter to specify whether you want to analyze real-time or historical results.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>/results/_topAnomalies?historical=false
  2. {
  3.   "size": 3,
  4.   "category_field": [
  5.     "ip"
  6.   ],
  7.   "order": "severity",
  8.   "task_id": "example-task-id",
  9.   "start_time_ms": 123456789000,
  10.   "end_time_ms": 987654321000
  11. }

Example response

  1. {
  2.   "buckets": [
  3.     {
  4.       "key": {
  5.         "ip": "1.2.3.4"
  6.       },
  7.       "doc_count": 10,
  8.       "max_anomaly_grade": 0.8
  9.     },
  10.     {
  11.       "key": {
  12.         "ip": "5.6.7.8"
  13.       },
  14.       "doc_count": 12,
  15.       "max_anomaly_grade": 0.6
  16.     },
  17.     {
  18.       "key": {
  19.         "ip": "9.10.11.12"
  20.       },
  21.       "doc_count": 3,
  22.       "max_anomaly_grade": 0.5
  23.     }
  24.   ]
  25. }

You can specify the following options.

OptionsDescriptionTypeRequired
sizeSpecify the number of top buckets that you want to see. Default is 10. The maximum number is 10,000.integerNo
category_fieldSpecify the set of category fields that you want to aggregate on. Defaults to all category fields for the detector.listNo
orderSpecify severity (anomaly grade) or occurrence (number of anomalies). Default is severity.stringNo
task_idSpecify a historical task ID to see results only from that specific task. Use only when historical=true, otherwise the anomaly detection plugin ignores this parameter.stringNo
start_time_msSpecify the time to start analyzing results, in Epoch milliseconds.longYes
end_time_msSpecify the time to end analyzing results, in Epoch milliseconds.longYes

Get detector stats

Introduced 1.0

Provides information about how the plugin is performing.

To get all stats:

Request

  1. GET _plugins/_anomaly_detection/stats

Example response

  1. {
  2.   "anomaly_detectors_index_status": "green",
  3.   "anomaly_detection_state_status": "green",
  4.   "single_entity_detector_count": 2,
  5.   "detector_count": 5,
  6.   "multi_entity_detector_count": 3,
  7.   "anomaly_detection_job_index_status": "green",
  8.   "models_checkpoint_index_status": "green",
  9.   "anomaly_results_index_status": "green",
  10.   "nodes": {
  11.     "2Z4q22BySEyzakYt_A0A2A": {
  12.       "ad_execute_request_count": 95,
  13.       "models": [
  14.         {
  15.           "detector_id": "WTBnTXwBjd8s6RK4b1Sz",
  16.           "model_type": "rcf",
  17.           "last_used_time": 1633398197185,
  18.           "model_id": "WTBnTXwBjd8s6RK4b1Sz_model_rcf_0",
  19.           "last_checkpoint_time": 1633396573679
  20.         },
  21.         ...
  22.       ],
  23.       "ad_canceled_batch_task_count": 0,
  24.       "ad_hc_execute_request_count": 75,
  25.       "ad_hc_execute_failure_count": 0,
  26.       "model_count": 28,
  27.       "ad_execute_failure_count": 1,
  28.       "ad_batch_task_failure_count": 0,
  29.       "ad_total_batch_task_execution_count": 27,
  30.       "ad_executing_batch_task_count": 3
  31.     },
  32.     "SWD7ihu9TaaW1zKwFZNVNg": {
  33.       "ad_execute_request_count": 12,
  34.       "models": [
  35.         {
  36.           "detector_id": "Zi5zTXwBwf_U8gjUTfJG",
  37.           "model_type": "entity",
  38.           "last_used_time": 1633398375008,
  39.           "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error13",
  40.           "last_checkpoint_time": 1633392973682,
  41.           "entity": [
  42.             {
  43.               "name": "error_type",
  44.               "value": "error13"
  45.             }
  46.           ]
  47.         },
  48.         ...
  49.       ],
  50.       "ad_canceled_batch_task_count": 1,
  51.       "ad_hc_execute_request_count": 0,
  52.       "ad_hc_execute_failure_count": 0,
  53.       "model_count": 15,
  54.       "ad_execute_failure_count": 2,
  55.       "ad_batch_task_failure_count": 0,
  56.       "ad_total_batch_task_execution_count": 27,
  57.       "ad_executing_batch_task_count": 4
  58.     },
  59.     "TQDUXEzyTJyV0H6_T4hYUw": {
  60.       "ad_execute_request_count": 0,
  61.       "models": [
  62.         {
  63.           "detector_id": "Zi5zTXwBwf_U8gjUTfJG",
  64.           "model_type": "entity",
  65.           "last_used_time": 1633398375004,
  66.           "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error24",
  67.           "last_checkpoint_time": 1633388177359,
  68.           "entity": [
  69.             {
  70.               "name": "error_type",
  71.               "value": "error24"
  72.             }
  73.           ]
  74.         },
  75.         ...
  76.       ],
  77.       "ad_canceled_batch_task_count": 0,
  78.       "ad_hc_execute_request_count": 0,
  79.       "ad_hc_execute_failure_count": 0,
  80.       "model_count": 22,
  81.       "ad_execute_failure_count": 0,
  82.       "ad_batch_task_failure_count": 0,
  83.       "ad_total_batch_task_execution_count": 28,
  84.       "ad_executing_batch_task_count": 3
  85.     }
  86.   }
  87. }

The model_count parameter shows the total number of models running on each node’s memory. For historical analysis, you see the values for the following fields:

  • ad_total_batch_task_execution_count
  • ad_executing_batch_task_count
  • ad_canceled_batch_task_count
  • ad_batch_task_failure_count

If haven’t run any historical analysis, these values show up as 0.

To get all stats for a specific node:

Request

  1. GET _plugins/_anomaly_detection/<nodeId>/stats

To get specific stats for a node:

Request

  1. GET _plugins/_anomaly_detection/<nodeId>/stats/<stat>

For example, to get the ad_execute_request_count value for node SWD7ihu9TaaW1zKwFZNVNg:

  1. GET _plugins/_anomaly_detection/SWD7ihu9TaaW1zKwFZNVNg/stats/ad_execute_request_count

Example response

  1. {
  2.   "nodes": {
  3.     "SWD7ihu9TaaW1zKwFZNVNg": {
  4.       "ad_execute_request_count": 12
  5.     }
  6.   }
  7. }

To get a specific type of stats:

Request

  1. GET _plugins/_anomaly_detection/stats/<stat>

For example:

  1. GET _plugins/_anomaly_detection/stats/ad_executing_batch_task_count

Example response

  1. {
  2.   "nodes": {
  3.     "2Z4q22BySEyzakYt_A0A2A": {
  4.       "ad_executing_batch_task_count": 3
  5.     },
  6.     "SWD7ihu9TaaW1zKwFZNVNg": {
  7.       "ad_executing_batch_task_count": 3
  8.     },
  9.     "TQDUXEzyTJyV0H6_T4hYUw": {
  10.       "ad_executing_batch_task_count": 4
  11.     }
  12.   }
  13. }

Profile detector

Introduced 1.0

Returns information related to the current state of the detector and memory usage, including current errors and shingle size, to help troubleshoot the detector.

This command helps locate logs by identifying the nodes that run the anomaly detector job for each detector.

It also helps track the initialization percentage, the required shingles, and the estimated time left.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/
  2. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true
  3. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/<type>
  4. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/<type1>,<type2>

Sample Responses

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile
  3. {
  4.   "state": "DISABLED",
  5.   "error": "Stopped detector: AD models memory usage exceeds our limit."
  6. }
  8. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true&pretty
  10. {
  11.   "state": "RUNNING",
  12.   "error": "",
  13.   "models": [
  14.     {
  15.       "model_id": "3Dh6TXwBwf_U8gjURE0F_entity_KSLSh0Wv05RQXiBAQHTEZg",
  16.       "entity": [
  17.         {
  18.           "name": "ip",
  19.           "value": "192.168.1.1"
  20.         },
  21.         {
  22.           "name": "error_type",
  23.           "value": "error8"
  24.         }
  25.       ],
  26.       "model_size_in_bytes": 403491,
  27.       "node_id": "2Z4q22BySEyzakYt_A0A2A"
  28.     },
  29.     ...
  30.   ],
  31.   "total_size_in_bytes": 12911712,
  32.   "init_progress": {
  33.     "percentage": "100%"
  34.   },
  35.   "total_entities": 33,
  36.   "active_entities": 32,
  37.   "ad_task": {
  38.     "ad_task": {
  39.       "task_id": "D3I5TnwBYwCbWecg7lN9",
  40.       "last_update_time": 1633399993685,
  41.       "started_by": "admin",
  42.       "state": "RUNNING",
  43.       "detector_id": "3Dh6TXwBwf_U8gjURE0F",
  44.       "task_progress": 0,
  45.       "init_progress": 0,
  46.       "execution_start_time": 1633399991933,
  47.       "is_latest": true,
  48.       "task_type": "HISTORICAL_HC_DETECTOR",
  49.       "coordinating_node": "2Z4q22BySEyzakYt_A0A2A",
  50.       "detector": {
  51.         "name": "testhc-mc",
  52.         "description": "test",
  53.         "time_field": "timestamp",
  54.         "indices": [
  55.           "server_log"
  56.         ],
  57.         "filter_query": {
  58.           "match_all": {
  59.             "boost": 1
  60.           }
  61.         },
  62.         "detection_interval": {
  63.           "period": {
  64.             "interval": 5,
  65.             "unit": "Minutes"
  66.           }
  67.         },
  68.         "window_delay": {
  69.           "period": {
  70.             "interval": 1,
  71.             "unit": "Minutes"
  72.           }
  73.         },
  74.         "shingle_size": 8,
  75.         "schema_version": 0,
  76.         "feature_attributes": [
  77.           {
  78.             "feature_id": "2zh6TXwBwf_U8gjUQ039",
  79.             "feature_name": "test",
  80.             "feature_enabled": true,
  81.             "aggregation_query": {
  82.               "test": {
  83.                 "sum": {
  84.                   "field": "value"
  85.                 }
  86.               }
  87.             }
  88.           }
  89.         ],
  90.         "ui_metadata": {
  91.           "features": {
  92.             "test": {
  93.               "aggregationBy": "sum",
  94.               "aggregationOf": "value",
  95.               "featureType": "simple_aggs"
  96.             }
  97.           },
  98.           "filters": []
  99.         },
  100.         "last_update_time": 1633387430916,
  101.         "category_field": [
  102.           "ip",
  103.           "error_type"
  104.         ],
  105.         "user": {
  106.           "name": "admin",
  107.           "backend_roles": [
  108.             "admin"
  109.           ],
  110.           "roles": [
  111.             "own_index",
  112.             "all_access"
  113.           ],
  114.           "custom_attribute_names": [],
  115.           "user_requested_tenant": "__user__"
  116.         },
  117.         "detector_type": "MULTI_ENTITY"
  118.       },
  119.       "detection_date_range": {
  120.         "start_time": 1632793800000,
  121.         "end_time": 1633398600000
  122.       },
  123.       "user": {
  124.         "name": "admin",
  125.         "backend_roles": [
  126.           "admin"
  127.         ],
  128.         "roles": [
  129.           "own_index",
  130.           "all_access"
  131.         ],
  132.         "custom_attribute_names": [],
  133.         "user_requested_tenant": "__user__"
  134.       }
  135.     },
  136.     "node_id": "2Z4q22BySEyzakYt_A0A2A",
  137.     "task_id": "D3I5TnwBYwCbWecg7lN9",
  138.     "task_type": "HISTORICAL_HC_DETECTOR",
  139.     "detector_task_slots": 10,
  140.     "total_entities_count": 32,
  141.     "pending_entities_count": 22,
  142.     "running_entities_count": 10,
  143.     "running_entities": [      """[{"name":"ip","value":"192.168.1.1"},{"name":"error_type","value":"error9"}]""",
  144.           ...],
  145.     "entity_task_profiles": [
  146.       {
  147.         "shingle_size": 8,
  148.         "rcf_total_updates": 1994,
  149.         "threshold_model_trained": true,
  150.         "threshold_model_training_data_size": 0,
  151.         "model_size_in_bytes": 1593240,
  152.         "node_id": "2Z4q22BySEyzakYt_A0A2A",
  153.         "entity": [
  154.           {
  155.             "name": "ip",
  156.             "value": "192.168.1.1"
  157.           },
  158.           {
  159.             "name": "error_type",
  160.             "value": "error7"
  161.           }
  162.         ],
  163.         "task_id": "E3I5TnwBYwCbWecg9FMm",
  164.         "task_type": "HISTORICAL_HC_ENTITY"
  165.       },
  166.       ...
  167.     ]
  168.   },
  169.   "model_count": 32
  170. }
  172. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/total_size_in_bytes
  174. {
  175.   "total_size_in_bytes": 13369344
  176. }

You can see the ad_task field only for historical analysis.

The model_count parameter shows the total number of models that a detector runs on each node’s memory. This is useful if you have several models running on your cluster and want to know the count.

If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory.

You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out.

For a single-entity detector:

Example response

  1. {
  2.   "state": "INIT",
  3.   "total_size_in_bytes": 0,
  4.   "init_progress": {
  5.     "percentage": "0%",
  6.     "needed_shingles": 128
  7.   },
  8.   "ad_task": {
  9.     "ad_task": {
  10.       "task_id": "cfUNOXwBFLNqSEcxAlde",
  11.       "last_update_time": 1633044731640,
  12.       "started_by": "admin",
  13.       "state": "RUNNING",
  14.       "detector_id": "qL4NOXwB__6eNorTAKtJ",
  15.       "task_progress": 0.49603173,
  16.       "init_progress": 1,
  17.       "current_piece": 1632739800000,
  18.       "execution_start_time": 1633044726365,
  19.       "is_latest": true,
  20.       "task_type": "HISTORICAL_SINGLE_ENTITY",
  21.       "coordinating_node": "bCtWtxWPThq0BIn5P5I4Xw",
  22.       "worker_node": "dIyavWhmSYWGz65b4u-lpQ",
  23.       "detector": {
  24.         "name": "detector1",
  25.         "description": "test",
  26.         "time_field": "timestamp",
  27.         "indices": [
  28.           "server_log"
  29.         ],
  30.         "filter_query": {
  31.           "match_all": {
  32.             "boost": 1
  33.           }
  34.         },
  35.         "detection_interval": {
  36.           "period": {
  37.             "interval": 5,
  38.             "unit": "Minutes"
  39.           }
  40.         },
  41.         "window_delay": {
  42.           "period": {
  43.             "interval": 1,
  44.             "unit": "Minutes"
  45.           }
  46.         },
  47.         "shingle_size": 8,
  48.         "schema_version": 0,
  49.         "feature_attributes": [
  50.           {
  51.             "feature_id": "p74NOXwB__6eNorTAKss",
  52.             "feature_name": "test-feature",
  53.             "feature_enabled": true,
  54.             "aggregation_query": {
  55.               "test_feature": {
  56.                 "sum": {
  57.                   "field": "value"
  58.                 }
  59.               }
  60.             }
  61.           }
  62.         ],
  63.         "ui_metadata": {
  64.           "features": {
  65.             "test-feature": {
  66.               "aggregationBy": "sum",
  67.               "aggregationOf": "value",
  68.               "featureType": "simple_aggs"
  69.             }
  70.           },
  71.           "filters": []
  72.         },
  73.         "last_update_time": 1633044725832,
  74.         "user": {
  75.           "name": "admin",
  76.           "backend_roles": [
  77.             "admin"
  78.           ],
  79.           "roles": [
  80.             "own_index",
  81.             "all_access"
  82.           ],
  83.           "custom_attribute_names": [],
  84.           "user_requested_tenant": "__user__"
  85.         },
  86.         "detector_type": "SINGLE_ENTITY"
  87.       },
  88.       "detection_date_range": {
  89.         "start_time": 1632439925885,
  90.         "end_time": 1633044725885
  91.       },
  92.       "user": {
  93.         "name": "admin",
  94.         "backend_roles": [
  95.           "admin"
  96.         ],
  97.         "roles": [
  98.           "own_index",
  99.           "all_access"
  100.         ],
  101.         "custom_attribute_names": [],
  102.         "user_requested_tenant": "__user__"
  103.       }
  104.     },
  105.     "shingle_size": 8,
  106.     "rcf_total_updates": 1994,
  107.     "threshold_model_trained": true,
  108.     "threshold_model_training_data_size": 0,
  109.     "model_size_in_bytes": 1593240,
  110.     "node_id": "dIyavWhmSYWGz65b4u-lpQ",
  111.     "detector_task_slots": 1
  112.   }
  113. }

The total_entities parameter shows you the total number of entities including the number of category fields for a detector.

Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000. For historical analysis, the anomaly detection plugin only detects the top 1,000 entities by default and caches the top entities in memory, so it doesn’t cost much to get the total count of entities for historical analysis.

The profile operation also provides information about each entity, such as the entity’s last_sample_timestamp and last_active_timestamp. last_sample_timestamp shows the last document in the input data source index containing the entity, while last_active_timestamp shows the timestamp when the entity’s model was last seen in the model cache.

If there are no anomaly results for an entity, either the entity doesn’t have any sample data or resources such as memory and disk IO are constrained relative to the number of entities.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true
  2. {
  3.   "entity": [
  4.     {
  5.       "name": "host",
  6.       "value": "i-00f28ec1eb8997686"
  7.     }
  8.   ]
  9. }

Sample Responses

  1. {
  2.   "is_active": true,
  3.   "last_active_timestamp": 1604026394879,
  4.   "last_sample_timestamp": 1604026394879,
  5.   "init_progress": {
  6.     "percentage": "100%"
  7.   },
  8.   "model": {
  9.     "model_id": "TFUdd3UBBwIAGQeRh5IS_entity_i-00f28ec1eb8997686",
  10.     "model_size_in_bytes": 712480,
  11.     "node_id": "MQ-bTBW3Q2uU_2zX3pyEQg"
  12.   },
  13.   "state": "RUNNING"
  14. }

To get profile information for only historical analysis, specify ad_task. Specifying _all is an expensive operation for multi-category high cardinality detectors.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all
  2. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/ad_task

Sample Responses

  1. {
  2.   "ad_task": {
  3.     "ad_task": {
  4.       "task_id": "CHI0TnwBYwCbWecgqgRA",
  5.       "last_update_time": 1633399648413,
  6.       "started_by": "admin",
  7.       "state": "RUNNING",
  8.       "detector_id": "3Dh6TXwBwf_U8gjURE0F",
  9.       "task_progress": 0,
  10.       "init_progress": 0,
  11.       "execution_start_time": 1633399646784,
  12.       "is_latest": true,
  13.       "task_type": "HISTORICAL_HC_DETECTOR",
  14.       "coordinating_node": "2Z4q22BySEyzakYt_A0A2A",
  15.       "detector": {
  16.         "name": "testhc-mc",
  17.         "description": "test",
  18.         "time_field": "timestamp",
  19.         "indices": [
  20.           "server_log"
  21.         ],
  22.         "filter_query": {
  23.           "match_all": {
  24.             "boost": 1
  25.           }
  26.         },
  27.         "detection_interval": {
  28.           "period": {
  29.             "interval": 5,
  30.             "unit": "Minutes"
  31.           }
  32.         },
  33.         "window_delay": {
  34.           "period": {
  35.             "interval": 1,
  36.             "unit": "Minutes"
  37.           }
  38.         },
  39.         "shingle_size": 8,
  40.         "schema_version": 0,
  41.         "feature_attributes": [
  42.           {
  43.             "feature_id": "2zh6TXwBwf_U8gjUQ039",
  44.             "feature_name": "test",
  45.             "feature_enabled": true,
  46.             "aggregation_query": {
  47.               "test": {
  48.                 "sum": {
  49.                   "field": "value"
  50.                 }
  51.               }
  52.             }
  53.           }
  54.         ],
  55.         "ui_metadata": {
  56.           "features": {
  57.             "test": {
  58.               "aggregationBy": "sum",
  59.               "aggregationOf": "value",
  60.               "featureType": "simple_aggs"
  61.             }
  62.           },
  63.           "filters": []
  64.         },
  65.         "last_update_time": 1633387430916,
  66.         "category_field": [
  67.           "ip",
  68.           "error_type"
  69.         ],
  70.         "user": {
  71.           "name": "admin",
  72.           "backend_roles": [
  73.             "admin"
  74.           ],
  75.           "roles": [
  76.             "own_index",
  77.             "all_access"
  78.           ],
  79.           "custom_attribute_names": [],
  80.           "user_requested_tenant": "__user__"
  81.         },
  82.         "detector_type": "MULTI_ENTITY"
  83.       },
  84.       "detection_date_range": {
  85.         "start_time": 1632793800000,
  86.         "end_time": 1633398600000
  87.       },
  88.       "user": {
  89.         "name": "admin",
  90.         "backend_roles": [
  91.           "admin"
  92.         ],
  93.         "roles": [
  94.           "own_index",
  95.           "all_access"
  96.         ],
  97.         "custom_attribute_names": [],
  98.         "user_requested_tenant": "__user__"
  99.       }
  100.     },
  101.     "node_id": "2Z4q22BySEyzakYt_A0A2A",
  102.     "task_id": "CHI0TnwBYwCbWecgqgRA",
  103.     "task_type": "HISTORICAL_HC_DETECTOR",
  104.     "detector_task_slots": 10,
  105.     "total_entities_count": 32,
  106.     "pending_entities_count": 22,
  107.     "running_entities_count": 10,
  108.     "running_entities" : [
  109.       """[{"name":"ip","value":"192.168.1.1"},{"name":"error_type","value":"error9"}]""",
  110.       ...
  111.     ],
  112.     "entity_task_profiles": [
  113.       {
  114.         "shingle_size": 8,
  115.         "rcf_total_updates": 994,
  116.         "threshold_model_trained": true,
  117.         "threshold_model_training_data_size": 0,
  118.         "model_size_in_bytes": 1593240,
  119.         "node_id": "2Z4q22BySEyzakYt_A0A2A",
  120.         "entity": [
  121.           {
  122.             "name": "ip",
  123.             "value": "192.168.1.1"
  124.           },
  125.           {
  126.             "name": "error_type",
  127.             "value": "error6"
  128.           }
  129.         ],
  130.         "task_id": "9XI0TnwBYwCbWecgsAd6",
  131.         "task_type": "HISTORICAL_HC_ENTITY"
  132.       },
  133.       ...
  134.     ]
  135.   }
  136. }

Delete detector results

Introduced 1.1

Deletes the results of a detector based on a query.

The delete detector results API only deletes anomaly result documents in the default result index. It doesn’t support deleting anomaly result documents stored in any custom result indices.

You need to manually delete anomaly result documents that you don’t need from custom result indices.

Request

  1. DELETE _plugins/_anomaly_detection/detectors/results
  2. {
  3.   "query": {
  4.     "bool": {
  5.       "filter": [
  6.         {
  7.           "term": {
  8.             "detector_id": {
  9.               "value": "rlDtOHwBD5tpxlbyW7Nt"
  10.             }
  11.           }
  12.         },
  13.         {
  14.           "term": {
  15.             "task_id": {
  16.               "value": "TM3tOHwBCi2h__AOXlyQ"
  17.             }
  18.           }
  19.         },
  20.         {
  21.           "range": {
  22.             "data_start_time": {
  23.               "lte": 1632441600000
  24.             }
  25.           }
  26.         }
  27.       ]
  28.     }
  29.   }
  30. }

Example response

  1. {
  2.   "took": 48,
  3.   "timed_out": false,
  4.   "total": 28,
  5.   "updated": 0,
  6.   "created": 0,
  7.   "deleted": 28,
  8.   "batches": 1,
  9.   "version_conflicts": 0,
  10.   "noops": 0,
  11.   "retries": {
  12.     "bulk": 0,
  13.     "search": 0
  14.   },
  15.   "throttled_millis": 0,
  16.   "requests_per_second": -1,
  17.   "throttled_until_millis": 0,
  18.   "failures": []
  19. }

Create monitor

Introduced 1.0

Create a monitor to set up alerts for the detector.

Request

  1. POST _plugins/_alerting/monitors
  2. {
  3.   "type": "monitor",
  4.   "name": "test-monitor",
  5.   "enabled": true,
  6.   "schedule": {
  7.     "period": {
  8.       "interval": 20,
  9.       "unit": "MINUTES"
  10.     }
  11.   },
  12.   "inputs": [
  13.     {
  14.       "search": {
  15.         "indices": [
  16.           ".opensearch-anomaly-results*"
  17.         ],
  18.         "query": {
  19.           "size": 1,
  20.           "query": {
  21.             "bool": {
  22.               "filter": [
  23.                 {
  24.                   "range": {
  25.                     "data_end_time": {
  26.                       "from": "||-20m",
  27.                       "to": "",
  28.                       "include_lower": true,
  29.                       "include_upper": true,
  30.                       "boost": 1
  31.                     }
  32.                   }
  33.                 },
  34.                 {
  35.                   "term": {
  36.                     "detector_id": {
  37.                       "value": "m4ccEnIBTXsGi3mvMt9p",
  38.                       "boost": 1
  39.                     }
  40.                   }
  41.                 }
  42.               ],
  43.               "adjust_pure_negative": true,
  44.               "boost": 1
  45.             }
  46.           },
  47.           "sort": [
  48.             {
  49.               "anomaly_grade": {
  50.                 "order": "desc"
  51.               }
  52.             },
  53.             {
  54.               "confidence": {
  55.                 "order": "desc"
  56.               }
  57.             }
  58.           ],
  59.           "aggregations": {
  60.             "max_anomaly_grade": {
  61.               "max": {
  62.                 "field": "anomaly_grade"
  63.               }
  64.             }
  65.           }
  66.         }
  67.       }
  68.     }
  69.   ],
  70.   "triggers": [
  71.     {
  72.       "name": "test-trigger",
  73.       "severity": "1",
  74.       "condition": {
  75.         "script": {
  76.           "source": "return ctx.results[0].aggregations.max_anomaly_grade.value != null && ctx.results[0].aggregations.max_anomaly_grade.value > 0.7 && ctx.results[0].hits.hits[0]._source.confidence > 0.7",
  77.           "lang": "painless"
  78.         }
  79.       },
  80.       "actions": [
  81.         {
  82.           "name": "test-action",
  83.           "destination_id": "ld7912sBlQ5JUWWFThoW",
  84.           "message_template": {
  85.             "source": "This is my message body."
  86.           },
  87.           "throttle_enabled": false,
  88.           "subject_template": {
  89.             "source": "TheSubject"
  90.           }
  91.         }
  92.       ]
  93.     }
  94.   ]
  95. }

Example response

  1. {
  2.   "_id": "OClTEnIBmSf7y6LP11Jz",
  3.   "_version": 1,
  4.   "_seq_no": 10,
  5.   "_primary_term": 1,
  6.   "monitor": {
  7.     "type": "monitor",
  8.     "schema_version": 1,
  9.     "name": "test-monitor",
  10.     "enabled": true,
  11.     "enabled_time": 1589445384043,
  12.     "schedule": {
  13.       "period": {
  14.         "interval": 20,
  15.         "unit": "MINUTES"
  16.       }
  17.     },
  18.     "inputs": [
  19.       {
  20.         "search": {
  21.           "indices": [
  22.             ".opensearch-anomaly-results*"
  23.           ],
  24.           "query": {
  25.             "size": 1,
  26.             "query": {
  27.               "bool": {
  28.                 "filter": [
  29.                   {
  30.                     "range": {
  31.                       "data_end_time": {
  32.                         "from": "||-20m",
  33.                         "to": "",
  34.                         "include_lower": true,
  35.                         "include_upper": true,
  36.                         "boost": 1
  37.                       }
  38.                     }
  39.                   },
  40.                   {
  41.                     "term": {
  42.                       "detector_id": {
  43.                         "value": "m4ccEnIBTXsGi3mvMt9p",
  44.                         "boost": 1
  45.                       }
  46.                     }
  47.                   }
  48.                 ],
  49.                 "adjust_pure_negative": true,
  50.                 "boost": 1
  51.               }
  52.             },
  53.             "sort": [
  54.               {
  55.                 "anomaly_grade": {
  56.                   "order": "desc"
  57.                 }
  58.               },
  59.               {
  60.                 "confidence": {
  61.                   "order": "desc"
  62.                 }
  63.               }
  64.             ],
  65.             "aggregations": {
  66.               "max_anomaly_grade": {
  67.                 "max": {
  68.                   "field": "anomaly_grade"
  69.                 }
  70.               }
  71.             }
  72.           }
  73.         }
  74.       }
  75.     ],
  76.     "triggers": [
  77.       {
  78.         "id": "NilTEnIBmSf7y6LP11Jr",
  79.         "name": "test-trigger",
  80.         "severity": "1",
  81.         "condition": {
  82.           "script": {
  83.             "source": "return ctx.results[0].aggregations.max_anomaly_grade.value != null && ctx.results[0].aggregations.max_anomaly_grade.value > 0.7 && ctx.results[0].hits.hits[0]._source.confidence > 0.7",
  84.             "lang": "painless"
  85.           }
  86.         },
  87.         "actions": [
  88.           {
  89.             "id": "NylTEnIBmSf7y6LP11Jr",
  90.             "name": "test-action",
  91.             "destination_id": "ld7912sBlQ5JUWWFThoW",
  92.             "message_template": {
  93.               "source": "This is my message body.",
  94.               "lang": "mustache"
  95.             },
  96.             "throttle_enabled": false,
  97.             "subject_template": {
  98.               "source": "TheSubject",
  99.               "lang": "mustache"
  100.             }
  101.           }
  102.         ]
  103.       }
  104.     ],
  105.     "last_update_time": 1589445384043
  106.   }
  107. }