Anomaly detection API

Use these anomaly detection operations to programmatically create and manage detectors.

Table of contents

Create anomaly detector

Creates an anomaly detector.

This command creates a detector named http_requests that finds anomalies based on the sum and average number of failed HTTP requests:

Request

  1. POST _plugins/_anomaly_detection/detectors
  2. {
  3.   "name": "test-detector",
  4.   "description": "Test detector",
  5.   "time_field": "timestamp",
  6.   "indices": [
  7.     "order*"
  8.   ],
  9.   "feature_attributes": [
  10.     {
  11.       "feature_name": "total_order",
  12.       "feature_enabled": true,
  13.       "aggregation_query": {
  14.         "total_order": {
  15.           "sum": {
  16.             "field": "value"
  17.           }
  18.         }
  19.       }
  20.     }
  21.   ],
  22.   "filter_query": {
  23.     "bool": {
  24.       "filter": [
  25.         {
  26.           "exists": {
  27.             "field": "value",
  28.             "boost": 1
  29.           }
  30.         }
  31.       ],
  32.       "adjust_pure_negative": true,
  33.       "boost": 1
  34.     }
  35.   },
  36.   "detection_interval": {
  37.     "period": {
  38.       "interval": 1,
  39.       "unit": "Minutes"
  40.     }
  41.   },
  42.   "window_delay": {
  43.     "period": {
  44.       "interval": 1,
  45.       "unit": "Minutes"
  46.     }
  47.   }
  48. }

Sample response

  1. {
  2.   "_id": "m4ccEnIBTXsGi3mvMt9p",
  3.   "_version": 1,
  4.   "_seq_no": 3,
  5.   "_primary_term": 1,
  6.   "anomaly_detector": {
  7.     "name": "test-detector",
  8.     "description": "Test detector",
  9.     "time_field": "timestamp",
  10.     "indices": [
  11.       "order*"
  12.     ],
  13.     "filter_query": {
  14.       "bool": {
  15.         "filter": [
  16.           {
  17.             "exists": {
  18.               "field": "value",
  19.               "boost": 1
  20.             }
  21.           }
  22.         ],
  23.         "adjust_pure_negative": true,
  24.         "boost": 1
  25.       }
  26.     },
  27.     "detection_interval": {
  28.       "period": {
  29.         "interval": 1,
  30.         "unit": "Minutes"
  31.       }
  32.     },
  33.     "window_delay": {
  34.       "period": {
  35.         "interval": 1,
  36.         "unit": "Minutes"
  37.       }
  38.     },
  39.     "schema_version": 0,
  40.     "feature_attributes": [
  41.       {
  42.         "feature_id": "mYccEnIBTXsGi3mvMd8_",
  43.         "feature_name": "total_order",
  44.         "feature_enabled": true,
  45.         "aggregation_query": {
  46.           "total_order": {
  47.             "sum": {
  48.               "field": "value"
  49.             }
  50.           }
  51.         }
  52.       }
  53.     ]
  54.   }
  55. }

To set a category field for high cardinality:

Request

  1. POST _plugins/_anomaly_detection/detectors
  2. {
  3.   "name": "Host OK Rate Detector",
  4.   "description": "ok rate",
  5.   "time_field": "@timestamp",
  6.   "indices": [
  7.     "host-cloudwatch"
  8.   ],
  9.   "category_field": [
  10.     "host"
  11.   ],
  12.   "feature_attributes": [
  13.     {
  14.       "feature_name": "latency_max",
  15.       "feature_enabled": true,
  16.       "aggregation_query": {
  17.         "latency_max": {
  18.           "max": {
  19.             "field": "latency"
  20.           }
  21.         }
  22.       }
  23.     }
  24.   ],
  25.   "window_delay": {
  26.     "period": {
  27.       "interval": 10,
  28.       "unit": "MINUTES"
  29.     }
  30.   },
  31.   "detection_interval": {
  32.     "period": {
  33.       "interval": 1,
  34.       "unit": "MINUTES"
  35.     }
  36.   }
  37. }

Sample response

  1. {
  2.   "_id": "4CIGoHUBTpMGN-4KzBQg",
  3.   "_version": 1,
  4.   "_seq_no": 0,
  5.   "anomaly_detector": {
  6.     "name": "Host OK Rate Detector",
  7.     "description": "ok rate",
  8.     "time_field": "@timestamp",
  9.     "indices": [
  10.       "server-metrics"
  11.     ],
  12.     "filter_query": {
  13.       "match_all": {
  14.         "boost": 1
  15.       }
  16.     },
  17.     "detection_interval": {
  18.       "period": {
  19.         "interval": 1,
  20.         "unit": "Minutes"
  21.       }
  22.     },
  23.     "window_delay": {
  24.       "period": {
  25.         "interval": 10,
  26.         "unit": "MINUTES"
  27.       }
  28.     },
  29.     "shingle_size": 1,
  30.     "schema_version": 2,
  31.     "feature_attributes": [
  32.       {
  33.         "feature_id": "0Kld3HUBhpHMyt2e_UHn",
  34.         "feature_name": "latency_max",
  35.         "feature_enabled": true,
  36.         "aggregation_query": {
  37.           "latency_max": {
  38.             "max": {
  39.               "field": "latency"
  40.             }
  41.           }
  42.         }
  43.       }
  44.     ],
  45.     "last_update_time": 1604707601438,
  46.     "category_field": [
  47.       "host"
  48.     ]
  49.   },
  50.   "_primary_term": 1
  51. }

To create a historical detector:

Request

  1. POST _plugins/_anomaly_detection/detectors
  2. {
  3.   "name": "test1",
  4.   "description": "test historical detector",
  5.   "time_field": "timestamp",
  6.   "indices": [
  7.     "host-cloudwatch"
  8.   ],
  9.   "filter_query": {
  10.     "match_all": {
  11.       "boost": 1
  12.     }
  13.   },
  14.   "detection_interval": {
  15.     "period": {
  16.       "interval": 1,
  17.       "unit": "Minutes"
  18.     }
  19.   },
  20.   "window_delay": {
  21.     "period": {
  22.       "interval": 1,
  23.       "unit": "Minutes"
  24.     }
  25.   },
  26.   "feature_attributes": [
  27.     {
  28.       "feature_name": "F1",
  29.       "feature_enabled": true,
  30.       "aggregation_query": {
  31.         "f_1": {
  32.           "sum": {
  33.             "field": "value"
  34.           }
  35.         }
  36.       }
  37.     }
  38.   ],
  39.   "detection_date_range": {
  40.     "start_time": 1577840401000,
  41.     "end_time": 1606121925000
  42.   }
  43. }

You can specify the following options.

OptionsDescriptionTypeRequired
nameThe name of the detector.stringYes
descriptionA description of the detector.stringYes
time_fieldThe name of the time field.stringYes
indicesA list of indices to use as the data source.listYes
feature_attributesSpecify a feature_name, set the enabled parameter to true, and specify an aggregation query.listYes
filter_queryProvide an optional filter query for your feature.objectNo
detection_intervalThe time interval for your anomaly detector.objectYes
window_delayAdd extra processing time for data collection.objectNo
category_fieldCategorizes or slices data with a dimension. Similar to GROUP BY in SQL.listNo
detection_date_rangeSpecify the start time and end time for a historical detector.objectNo

Preview detector

Passes a date range to the anomaly detector to return any anomalies within that date range.

Request

  1. POST _plugins/_anomaly_detection/detectors/<detectorId>/_preview
  2. {
  3.   "period_start": 1588838250000,
  4.   "period_end": 1589443050000
  5. }

Sample response

  1. {
  2.   "anomaly_result": [
  3.     ...
  4.     {
  5.       "detector_id": "m4ccEnIBTXsGi3mvMt9p",
  6.       "data_start_time": 1588843020000,
  7.       "data_end_time": 1588843620000,
  8.       "feature_data": [
  9.         {
  10.           "feature_id": "xxokEnIBcpeWMD987A1X",
  11.           "feature_name": "total_order",
  12.           "data": 489.9929131106
  13.         }
  14.       ],
  15.       "anomaly_grade": 0,
  16.       "confidence": 0.99
  17.     }
  18.     ...
  19.   ],
  20.   "anomaly_detector": {
  21.     "name": "test-detector",
  22.     "description": "Test detector",
  23.     "time_field": "timestamp",
  24.     "indices": [
  25.       "order*"
  26.     ],
  27.     "filter_query": {
  28.       "bool": {
  29.         "filter": [
  30.           {
  31.             "exists": {
  32.               "field": "value",
  33.               "boost": 1
  34.             }
  35.           }
  36.         ],
  37.         "adjust_pure_negative": true,
  38.         "boost": 1
  39.       }
  40.     },
  41.     "detection_interval": {
  42.       "period": {
  43.         "interval": 10,
  44.         "unit": "MINUTES"
  45.       }
  46.     },
  47.     "window_delay": {
  48.       "period": {
  49.         "interval": 1,
  50.         "unit": "MINUTES"
  51.       }
  52.     },
  53.     "schema_version": 0,
  54.     "feature_attributes": [
  55.       {
  56.         "feature_id": "xxokEnIBcpeWMD987A1X",
  57.         "feature_name": "total_order",
  58.         "feature_enabled": true,
  59.         "aggregation_query": {
  60.           "total_order": {
  61.             "sum": {
  62.               "field": "value"
  63.             }
  64.           }
  65.         }
  66.       }
  67.     ],
  68.     "last_update_time": 1589442309241
  69.   }
  70. }

If you specify a category field, each result is associated with an entity:

Sample response

  1. {
  2.   "anomaly_result": [
  3.     {
  4.       "detector_id": "4CIGoHUBTpMGN-4KzBQg",
  5.       "data_start_time": 1604277960000,
  6.       "data_end_time": 1604278020000,
  7.       "schema_version": 0,
  8.       "anomaly_grade": 0,
  9.       "confidence": 0.99
  10.     }
  11.   ],
  12.   "entity": [
  13.     {
  14.       "name": "host",
  15.       "value": "i-00f28ec1eb8997686"
  16.     }
  17.   ]
  18. },
  19. {
  20.   "detector_id": "4CIGoHUBTpMGN-4KzBQg",
  21.   "data_start_time": 1604278020000,
  22.   "data_end_time": 1604278080000,
  23.   "schema_version": 0,
  24.   "feature_data": [
  25.     {
  26.       "feature_id": "0Kld3HUBhpHMyt2e_UHn",
  27.       "feature_name": "latency_max",
  28.       "data": -17
  29.     }
  30.   ],
  31.   "anomaly_grade": 0,
  32.   "confidence": 0.99,
  33.   "entity": [
  34.     {
  35.       "name": "host",
  36.       "value": "i-00f28ec1eb8997686"
  37.     }
  38.   ]
  39. }
  40. ...

Start detector job

Starts a real-time or historical anomaly detector job.

Request

  1. POST _plugins/_anomaly_detection/detectors/<detectorId>/_start

Sample response

  1. {
  2.   "_id" : "m4ccEnIBTXsGi3mvMt9p",
  3.   "_version" : 1,
  4.   "_seq_no" : 6,
  5.   "_primary_term" : 1
  6. }

Stop detector job

Stops a real-time or historical anomaly detector job.

Request

  1. POST _plugins/_anomaly_detection/detectors/<detectorId>/_stop

Sample response

  1. Stopped detector: m4ccEnIBTXsGi3mvMt9p

Search detector result

Returns all results for a search query.

Request

  1. GET _plugins/_anomaly_detection/detectors/results/_search
  2. POST _plugins/_anomaly_detection/detectors/results/_search
  4. {
  5.   "query": {
  6.     "bool": {
  7.       "must": {
  8.         "range": {
  9.           "anomaly_score": {
  10.             "gte": 0.6,
  11.             "lte": 1
  12.           }
  13.         }
  14.       }
  15.     }
  16.   }
  17. }

Sample response

  1. {
  2.   "took": 9,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 25,
  6.     "successful": 25,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 2,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": 1,
  16.     "hits": [
  17.       {
  18.         "_index": ".opendistro-anomaly-results-history-2020.04.30-1",
  19.         "_type": "_doc",
  20.         "_id": "_KBrzXEBbpoKkFM5mStm",
  21.         "_version": 1,
  22.         "_seq_no": 58,
  23.         "_primary_term": 1,
  24.         "_score": 1,
  25.         "_source": {
  26.           "detector_id": "2KDozHEBbpoKkFM58yr6",
  27.           "anomaly_score": 0.8995068350366767,
  28.           "execution_start_time": 1588289313114,
  29.           "data_end_time": 1588289313114,
  30.           "confidence": 0.84214852704501,
  31.           "data_start_time": 1588289253114,
  32.           "feature_data": [
  33.             {
  34.               "feature_id": "X0fpzHEB5NGZmIRkXKcy",
  35.               "feature_name": "total_error",
  36.               "data": 20
  37.             }
  38.           ],
  39.           "execution_end_time": 1588289313126,
  40.           "anomaly_grade": 0
  41.         }
  42.       },
  43.       {
  44.         "_index": ".opendistro-anomaly-results-history-2020.04.30-1",
  45.         "_type": "_doc",
  46.         "_id": "EqB1zXEBbpoKkFM5qyyE",
  47.         "_version": 1,
  48.         "_seq_no": 61,
  49.         "_primary_term": 1,
  50.         "_score": 1,
  51.         "_source": {
  52.           "detector_id": "2KDozHEBbpoKkFM58yr6",
  53.           "anomaly_score": 0.7086834513354907,
  54.           "execution_start_time": 1588289973113,
  55.           "data_end_time": 1588289973113,
  56.           "confidence": 0.42162017029510446,
  57.           "data_start_time": 1588289913113,
  58.           "feature_data": [
  59.             {
  60.               "feature_id": "X0fpzHEB5NGZmIRkXKcy",
  61.               "feature_name": "memory_usage",
  62.               "data": 20.0347333108
  63.             }
  64.           ],
  65.           "execution_end_time": 1588289973124,
  66.           "anomaly_grade": 0
  67.         }
  68.       }
  69.     ]
  70.   }
  71. }

In high cardinality detectors, the result contains entity information.

To see an ordered set of anomaly records for an entity with an anomaly within a certain time range for a specific feature value:

Request

  1. POST _plugins/_anomaly_detection/detectors/results/_search
  2. {
  3.   "query": {
  4.     "bool": {
  5.       "filter": [
  6.         {
  7.           "term": {
  8.             "detector_id": "4CIGoHUBTpMGN-4KzBQg"
  9.           }
  10.         },
  11.         {
  12.           "range": {
  13.             "anomaly_grade": {
  14.               "gt": 0
  15.             }
  16.           }
  17.         },
  18.         {
  19.           "nested": {
  20.             "path": "entity",
  21.             "query": {
  22.               "bool": {
  23.                 "must": [
  24.                   {
  25.                     "term": {
  26.                       "entity.value": "i-00f28ec1eb8997685"
  27.                     }
  28.                   }
  29.                 ]
  30.               }
  31.             }
  32.           }
  33.         }
  34.       ]
  35.     }
  36.   },
  37.   "size": 8,
  38.   "sort": [
  39.     {
  40.       "execution_end_time": {
  41.         "order": "desc"
  42.       }
  43.     }
  44.   ],
  45.   "track_total_hits": true
  46. }

Sample response

  1. {
  2.   "took": 443,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 1,
  6.     "successful": 1,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 7,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": null,
  16.     "hits": [
  17.       {
  18.         "_index": ".opendistro-anomaly-results-history-2020.11.07-1",
  19.         "_type": "_doc",
  20.         "_id": "BiItoHUBTpMGN-4KARY5",
  21.         "_version": 1,
  22.         "_seq_no": 206,
  23.         "_primary_term": 1,
  24.         "_score": null,
  25.         "_source": {
  26.           "detector_id": "4CIGoHUBTpMGN-4KzBQg",
  27.           "schema_version": 2,
  28.           "anomaly_score": 2.462550517055763,
  29.           "execution_start_time": 1604710105400,
  30.           "data_end_time": 1604710094516,
  31.           "confidence": 0.8246254862573076,
  32.           "data_start_time": 1604710034516,
  33.           "feature_data": [
  34.             {
  35.               "feature_id": "0Kld3HUBhpHMyt2e_UHn",
  36.               "feature_name": "latency_max",
  37.               "data": 3526
  38.             }
  39.           ],
  40.           "execution_end_time": 1604710105401,
  41.           "anomaly_grade": 0.08045977011494891,
  42.           "entity": [
  43.             {
  44.               "name": "host",
  45.               "value": "i-00f28ec1eb8997685"
  46.             }
  47.           ]
  48.         },
  49.         "sort": [
  50.           1604710105401
  51.         ]
  52.       },
  53.       {
  54.         "_index": ".opendistro-anomaly-results-history-2020.11.07-1",
  55.         "_type": "_doc",
  56.         "_id": "wiImoHUBTpMGN-4KlhXs",
  57.         "_version": 1,
  58.         "_seq_no": 156,
  59.         "_primary_term": 1,
  60.         "_score": null,
  61.         "_source": {
  62.           "detector_id": "4CIGoHUBTpMGN-4KzBQg",
  63.           "schema_version": 2,
  64.           "anomaly_score": 4.892453213261217,
  65.           "execution_start_time": 1604709684971,
  66.           "data_end_time": 1604709674522,
  67.           "confidence": 0.8313735633713821,
  68.           "data_start_time": 1604709614522,
  69.           "feature_data": [
  70.             {
  71.               "feature_id": "0Kld3HUBhpHMyt2e_UHn",
  72.               "feature_name": "latency_max",
  73.               "data": 5709
  74.             }
  75.           ],
  76.           "execution_end_time": 1604709684971,
  77.           "anomaly_grade": 0.06542056074767538,
  78.           "entity": [
  79.             {
  80.               "name": "host",
  81.               "value": "i-00f28ec1eb8997685"
  82.             }
  83.           ]
  84.         },
  85.         "sort": [
  86.           1604709684971
  87.         ]
  88.       },
  89.       {
  90.         "_index": ".opendistro-anomaly-results-history-2020.11.07-1",
  91.         "_type": "_doc",
  92.         "_id": "ZiIcoHUBTpMGN-4KhhVA",
  93.         "_version": 1,
  94.         "_seq_no": 79,
  95.         "_primary_term": 1,
  96.         "_score": null,
  97.         "_source": {
  98.           "detector_id": "4CIGoHUBTpMGN-4KzBQg",
  99.           "schema_version": 2,
  100.           "anomaly_score": 3.187717536855158,
  101.           "execution_start_time": 1604709025343,
  102.           "data_end_time": 1604709014520,
  103.           "confidence": 0.8301116064308817,
  104.           "data_start_time": 1604708954520,
  105.           "feature_data": [
  106.             {
  107.               "feature_id": "0Kld3HUBhpHMyt2e_UHn",
  108.               "feature_name": "latency_max",
  109.               "data": 441
  110.             }
  111.           ],
  112.           "execution_end_time": 1604709025344,
  113.           "anomaly_grade": 0.040767386091133916,
  114.           "entity": [
  115.             {
  116.               "name": "host",
  117.               "value": "i-00f28ec1eb8997685"
  118.             }
  119.           ]
  120.         },
  121.         "sort": [
  122.           1604709025344
  123.         ]
  124.       }
  125.     ]
  126.   }
  127. }

In historical detectors, specify the detector_id. To get the latest task:

Request

  1. GET _plugins/_anomaly_detection/detectors/<detector_id>?task=true

To query the anomaly results with task_id:

Request

  1. GET _plugins/_anomaly_detection/detectors/results/_search
  2. {
  3.   "query": {
  4.     "term": {
  5.       "task_id": {
  6.         "value": "NnlV9HUBQxqfQ7vBJNzy"
  7.       }
  8.     }
  9.   }
  10. }

Sample response

  1. {
  2.   "took": 1,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 1,
  6.     "successful": 1,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 1,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": 2.1366,
  16.     "hits": [
  17.       {
  18.         "_index": ".opendistro-anomaly-detection-state",
  19.         "_type": "_doc",
  20.         "_id": "CoM8WncBtt2qvI-LZO7_",
  21.         "_version": 8,
  22.         "_seq_no": 1351,
  23.         "_primary_term": 3,
  24.         "_score": 2.1366,
  25.         "_source": {
  26.           "detector_id": "dZc8WncBgO2zoQoFWVBA",
  27.           "worker_node": "dk6-HuKQRMKm2fi8TSDHsg",
  28.           "task_progress": 0.09486946,
  29.           "last_update_time": 1612126667008,
  30.           "execution_start_time": 1612126643455,
  31.           "state": "RUNNING",
  32.           "coordinating_node": "gs213KqjS4q7H4Bmn_ZuLA",
  33.           "current_piece": 1583503800000,
  34.           "task_type": "HISTORICAL",
  35.           "started_by": "admin",
  36.           "init_progress": 1,
  37.           "is_latest": true,
  38.           "detector": {
  39.             "description": "test",
  40.             "ui_metadata": {
  41.               "features": {
  42.                 "F1": {
  43.                   "aggregationBy": "sum",
  44.                   "aggregationOf": "value",
  45.                   "featureType": "simple_aggs"
  46.                 }
  47.               }
  48.             },
  49.             "detection_date_range": {
  50.               "start_time": 1580504240308,
  51.               "end_time": 1612126640308
  52.             },
  53.             "feature_attributes": [
  54.               {
  55.                 "feature_id": "dJc8WncBgO2zoQoFWVAt",
  56.                 "feature_enabled": true,
  57.                 "feature_name": "F1",
  58.                 "aggregation_query": {
  59.                   "f_1": {
  60.                     "sum": {
  61.                       "field": "value"
  62.                     }
  63.                   }
  64.                 }
  65.               }
  66.             ],
  67.             "schema_version": 0,
  68.             "time_field": "timestamp",
  69.             "last_update_time": 1612126640448,
  70.             "indices": [
  71.               "nab_art_daily_jumpsdown"
  72.             ],
  73.             "window_delay": {
  74.               "period": {
  75.                 "unit": "Minutes",
  76.                 "interval": 1
  77.               }
  78.             },
  79.             "detection_interval": {
  80.               "period": {
  81.                 "unit": "Minutes",
  82.                 "interval": 10
  83.               }
  84.             },
  85.             "name": "test-historical-detector",
  86.             "filter_query": {
  87.               "match_all": {
  88.                 "boost": 1
  89.               }
  90.             },
  91.             "shingle_size": 8,
  92.             "user": {
  93.               "backend_roles": [
  94.                 "admin"
  95.               ],
  96.               "custom_attribute_names": [],
  97.               "roles": [
  98.                 "all_access",
  99.                 "own_index"
  100.               ],
  101.               "name": "admin",
  102.               "user_requested_tenant": "__user__"
  103.             },
  104.             "detector_type": "HISTORICAL_SINGLE_ENTITY"
  105.           },
  106.           "user": {
  107.             "backend_roles": [
  108.               "admin"
  109.             ],
  110.             "custom_attribute_names": [],
  111.             "roles": [
  112.               "all_access",
  113.               "own_index"
  114.             ],
  115.             "name": "admin",
  116.             "user_requested_tenant": "__user__"
  117.           }
  118.         }
  119.       }
  120.     ]
  121.   }
  122. }

Delete detector

Deletes a detector based on the detector_id. To delete a detector, you need to first stop the detector.

Request

  1. DELETE _plugins/_anomaly_detection/detectors/<detectorId>

Sample response

  1. {
  2.   "_index" : ".opendistro-anomaly-detectors",
  3.   "_type" : "_doc",
  4.   "_id" : "m4ccEnIBTXsGi3mvMt9p",
  5.   "_version" : 2,
  6.   "result" : "deleted",
  7.   "forced_refresh" : true,
  8.   "_shards" : {
  9.     "total" : 2,
  10.     "successful" : 2,
  11.     "failed" : 0
  12.   },
  13.   "_seq_no" : 6,
  14.   "_primary_term" : 1
  15. }

Update detector

Updates a detector with any changes, including the description or adding or removing of features. To update a detector, you need to first stop the detector.

Request

  1. PUT _plugins/_anomaly_detection/detectors/<detectorId>
  2. {
  3.   "name": "test-detector",
  4.   "description": "Test detector",
  5.   "time_field": "timestamp",
  6.   "indices": [
  7.     "order*"
  8.   ],
  9.   "feature_attributes": [
  10.     {
  11.       "feature_name": "total_order",
  12.       "feature_enabled": true,
  13.       "aggregation_query": {
  14.         "total_order": {
  15.           "sum": {
  16.             "field": "value"
  17.           }
  18.         }
  19.       }
  20.     }
  21.   ],
  22.   "filter_query": {
  23.     "bool": {
  24.       "filter": [
  25.         {
  26.           "exists": {
  27.             "field": "value",
  28.             "boost": 1
  29.           }
  30.         }
  31.       ],
  32.       "adjust_pure_negative": true,
  33.       "boost": 1
  34.     }
  35.   },
  36.   "detection_interval": {
  37.     "period": {
  38.       "interval": 10,
  39.       "unit": "MINUTES"
  40.     }
  41.   },
  42.   "window_delay": {
  43.     "period": {
  44.       "interval": 1,
  45.       "unit": "MINUTES"
  46.     }
  47.   }
  48. }

Sample response

  1. {
  2.   "_id" : "m4ccEnIBTXsGi3mvMt9p",
  3.   "_version" : 2,
  4.   "_seq_no" : 4,
  5.   "_primary_term" : 1,
  6.   "anomaly_detector" : {
  7.     "name" : "test-detector",
  8.     "description" : "Test detector",
  9.     "time_field" : "timestamp",
  10.     "indices" : [
  11.       "order*"
  12.     ],
  13.     "filter_query" : {
  14.       "bool" : {
  15.         "filter" : [
  16.           {
  17.             "exists" : {
  18.               "field" : "value",
  19.               "boost" : 1.0
  20.             }
  21.           }
  22.         ],
  23.         "adjust_pure_negative" : true,
  24.         "boost" : 1.0
  25.       }
  26.     },
  27.     "detection_interval" : {
  28.       "period" : {
  29.         "interval" : 10,
  30.         "unit" : "Minutes"
  31.       }
  32.     },
  33.     "window_delay" : {
  34.       "period" : {
  35.         "interval" : 1,
  36.         "unit" : "Minutes"
  37.       }
  38.     },
  39.     "schema_version" : 0,
  40.     "feature_attributes" : [
  41.       {
  42.         "feature_id" : "xxokEnIBcpeWMD987A1X",
  43.         "feature_name" : "total_order",
  44.         "feature_enabled" : true,
  45.         "aggregation_query" : {
  46.           "total_order" : {
  47.             "sum" : {
  48.               "field" : "value"
  49.             }
  50.           }
  51.         }
  52.       }
  53.     ]
  54.   }
  55. }

To update a historical detector:

Request

  1. PUT _plugins/_anomaly_detection/detectors/<detectorId>
  2. {
  3.   "name": "test1",
  4.   "description": "test historical detector",
  5.   "time_field": "timestamp",
  6.   "indices": [
  7.     "nab_art_daily_jumpsdown"
  8.   ],
  9.   "filter_query": {
  10.     "match_all": {
  11.       "boost": 1
  12.     }
  13.   },
  14.   "detection_interval": {
  15.     "period": {
  16.       "interval": 1,
  17.       "unit": "Minutes"
  18.     }
  19.   },
  20.   "window_delay": {
  21.     "period": {
  22.       "interval": 1,
  23.       "unit": "Minutes"
  24.     }
  25.   },
  26.   "feature_attributes": [
  27.     {
  28.       "feature_name": "F1",
  29.       "feature_enabled": true,
  30.       "aggregation_query": {
  31.         "f_1": {
  32.           "sum": {
  33.             "field": "value"
  34.           }
  35.         }
  36.       }
  37.     }
  38.   ],
  39.   "detection_date_range": {
  40.     "start_time": 1577840401000,
  41.     "end_time": 1606121925000
  42.   }
  43. }

Get detector

Returns all information about a detector based on the detector_id.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>

Sample response

  1. {
  2.   "_id" : "m4ccEnIBTXsGi3mvMt9p",
  3.   "_version" : 1,
  4.   "_primary_term" : 1,
  5.   "_seq_no" : 3,
  6.   "anomaly_detector" : {
  7.     "name" : "test-detector",
  8.     "description" : "Test detector",
  9.     "time_field" : "timestamp",
  10.     "indices" : [
  11.       "order*"
  12.     ],
  13.     "filter_query" : {
  14.       "bool" : {
  15.         "filter" : [
  16.           {
  17.             "exists" : {
  18.               "field" : "value",
  19.               "boost" : 1.0
  20.             }
  21.           }
  22.         ],
  23.         "adjust_pure_negative" : true,
  24.         "boost" : 1.0
  25.       }
  26.     },
  27.     "detection_interval" : {
  28.       "period" : {
  29.         "interval" : 1,
  30.         "unit" : "Minutes"
  31.       }
  32.     },
  33.     "window_delay" : {
  34.       "period" : {
  35.         "interval" : 1,
  36.         "unit" : "Minutes"
  37.       }
  38.     },
  39.     "schema_version" : 0,
  40.     "feature_attributes" : [
  41.       {
  42.         "feature_id" : "mYccEnIBTXsGi3mvMd8_",
  43.         "feature_name" : "total_order",
  44.         "feature_enabled" : true,
  45.         "aggregation_query" : {
  46.           "total_order" : {
  47.             "sum" : {
  48.               "field" : "value"
  49.             }
  50.           }
  51.         }
  52.       }
  53.     ],
  54.     "last_update_time" : 1589441737319
  55.   }
  56. }

Use job=true to get anomaly detection job information.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>?job=true

Sample response

  1. {
  2.   "_id" : "m4ccEnIBTXsGi3mvMt9p",
  3.   "_version" : 1,
  4.   "_primary_term" : 1,
  5.   "_seq_no" : 3,
  6.   "anomaly_detector" : {
  7.     "name" : "test-detector",
  8.     "description" : "Test detector",
  9.     "time_field" : "timestamp",
  10.     "indices" : [
  11.       "order*"
  12.     ],
  13.     "filter_query" : {
  14.       "bool" : {
  15.         "filter" : [
  16.           {
  17.             "exists" : {
  18.               "field" : "value",
  19.               "boost" : 1.0
  20.             }
  21.           }
  22.         ],
  23.         "adjust_pure_negative" : true,
  24.         "boost" : 1.0
  25.       }
  26.     },
  27.     "detection_interval" : {
  28.       "period" : {
  29.         "interval" : 1,
  30.         "unit" : "Minutes"
  31.       }
  32.     },
  33.     "window_delay" : {
  34.       "period" : {
  35.         "interval" : 1,
  36.         "unit" : "Minutes"
  37.       }
  38.     },
  39.     "schema_version" : 0,
  40.     "feature_attributes" : [
  41.       {
  42.         "feature_id" : "mYccEnIBTXsGi3mvMd8_",
  43.         "feature_name" : "total_order",
  44.         "feature_enabled" : true,
  45.         "aggregation_query" : {
  46.           "total_order" : {
  47.             "sum" : {
  48.               "field" : "value"
  49.             }
  50.           }
  51.         }
  52.       }
  53.     ],
  54.     "last_update_time" : 1589441737319
  55.   },
  56.   "anomaly_detector_job" : {
  57.     "name" : "m4ccEnIBTXsGi3mvMt9p",
  58.     "schedule" : {
  59.       "interval" : {
  60.         "start_time" : 1589442051271,
  61.         "period" : 1,
  62.         "unit" : "Minutes"
  63.       }
  64.     },
  65.     "window_delay" : {
  66.       "period" : {
  67.         "interval" : 1,
  68.         "unit" : "Minutes"
  69.       }
  70.     },
  71.     "enabled" : true,
  72.     "enabled_time" : 1589442051271,
  73.     "last_update_time" : 1589442051271,
  74.     "lock_duration_seconds" : 60
  75.   }
  76. }

Use task=true to get historical detector task information.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>?task=true

Sample response

  1. {
  2.   "_id": "BwzKQXcB89DLS7G9rg7Y",
  3.   "_version": 1,
  4.   "_primary_term": 2,
  5.   "_seq_no": 10,
  6.   "anomaly_detector": {
  7.     "name": "test-ylwu1",
  8.     "description": "test",
  9.     "time_field": "timestamp",
  10.     "indices": [
  11.       "nab*"
  12.     ],
  13.     "filter_query": {
  14.       "match_all": {
  15.         "boost": 1
  16.       }
  17.     },
  18.     "detection_interval": {
  19.       "period": {
  20.         "interval": 10,
  21.         "unit": "Minutes"
  22.       }
  23.     },
  24.     "window_delay": {
  25.       "period": {
  26.         "interval": 1,
  27.         "unit": "Minutes"
  28.       }
  29.     },
  30.     "shingle_size": 8,
  31.     "schema_version": 0,
  32.     "feature_attributes": [
  33.       {
  34.         "feature_id": "BgzKQXcB89DLS7G9rg7G",
  35.         "feature_name": "F1",
  36.         "feature_enabled": true,
  37.         "aggregation_query": {
  38.           "f_1": {
  39.             "sum": {
  40.               "field": "value"
  41.             }
  42.           }
  43.         }
  44.       }
  45.     ],
  46.     "ui_metadata": {
  47.       "features": {
  48.         "F1": {
  49.           "aggregationBy": "sum",
  50.           "aggregationOf": "value",
  51.           "featureType": "simple_aggs"
  52.         }
  53.       }
  54.     },
  55.     "last_update_time": 1611716538071,
  56.     "user": {
  57.       "name": "admin",
  58.       "backend_roles": [
  59.         "admin"
  60.       ],
  61.       "roles": [
  62.         "all_access",
  63.         "own_index"
  64.       ],
  65.       "custom_attribute_names": [],
  66.       "user_requested_tenant": "__user__"
  67.     },
  68.     "detector_type": "HISTORICAL_SINGLE_ENTITY",
  69.     "detection_date_range": {
  70.       "start_time": 1580094137997,
  71.       "end_time": 1611716537997
  72.     }
  73.   },
  74.   "anomaly_detection_task": {
  75.     "task_id": "sgxaRXcB89DLS7G9RfIO",
  76.     "last_update_time": 1611776648699,
  77.     "started_by": "admin",
  78.     "state": "FINISHED",
  79.     "detector_id": "BwzKQXcB89DLS7G9rg7Y",
  80.     "task_progress": 1,
  81.     "init_progress": 1,
  82.     "current_piece": 1611716400000,
  83.     "execution_start_time": 1611776279822,
  84.     "execution_end_time": 1611776648679,
  85.     "is_latest": true,
  86.     "task_type": "HISTORICAL",
  87.     "coordinating_node": "gs213KqjS4q7H4Bmn_ZuLA",
  88.     "worker_node": "PgfR3JhbT7yJMx7bwQ6E3w",
  89.     "detector": {
  90.       "name": "test-ylwu1",
  91.       "description": "test",
  92.       "time_field": "timestamp",
  93.       "indices": [
  94.         "nab*"
  95.       ],
  96.       "filter_query": {
  97.         "match_all": {
  98.           "boost": 1
  99.         }
  100.       },
  101.       "detection_interval": {
  102.         "period": {
  103.           "interval": 10,
  104.           "unit": "Minutes"
  105.         }
  106.       },
  107.       "window_delay": {
  108.         "period": {
  109.           "interval": 1,
  110.           "unit": "Minutes"
  111.         }
  112.       },
  113.       "shingle_size": 8,
  114.       "schema_version": 0,
  115.       "feature_attributes": [
  116.         {
  117.           "feature_id": "BgzKQXcB89DLS7G9rg7G",
  118.           "feature_name": "F1",
  119.           "feature_enabled": true,
  120.           "aggregation_query": {
  121.             "f_1": {
  122.               "sum": {
  123.                 "field": "value"
  124.               }
  125.             }
  126.           }
  127.         }
  128.       ],
  129.       "ui_metadata": {
  130.         "features": {
  131.           "F1": {
  132.             "aggregationBy": "sum",
  133.             "aggregationOf": "value",
  134.             "featureType": "simple_aggs"
  135.           }
  136.         }
  137.       },
  138.       "last_update_time": 1611716538071,
  139.       "user": {
  140.         "name": "admin",
  141.         "backend_roles": [
  142.           "admin"
  143.         ],
  144.         "roles": [
  145.           "all_access",
  146.           "own_index"
  147.         ],
  148.         "custom_attribute_names": [],
  149.         "user_requested_tenant": "__user__"
  150.       },
  151.       "detector_type": "HISTORICAL_SINGLE_ENTITY",
  152.       "detection_date_range": {
  153.         "start_time": 1580094137997,
  154.         "end_time": 1611716537997
  155.       }
  156.     },
  157.     "user": {
  158.       "name": "admin",
  159.       "backend_roles": [
  160.         "admin"
  161.       ],
  162.       "roles": [
  163.         "all_access",
  164.         "own_index"
  165.       ],
  166.       "custom_attribute_names": [],
  167.       "user_requested_tenant": "__user__"
  168.     }
  169.   }
  170. }

Search detector

Returns all anomaly detectors for a search query.

Request

  1. GET _plugins/_anomaly_detection/detectors/_search
  2. POST _plugins/_anomaly_detection/detectors/_search
  4. Sample Input:
  5. {
  6.   "query": {
  7.     "match": {
  8.       "name": "test-detector"
  9.     }
  10.   }
  11. }

Sample response

  1. {
  2.   "took": 13,
  3.   "timed_out": false,
  4.   "_shards": {
  5.     "total": 5,
  6.     "successful": 5,
  7.     "skipped": 0,
  8.     "failed": 0
  9.   },
  10.   "hits": {
  11.     "total": {
  12.       "value": 994,
  13.       "relation": "eq"
  14.     },
  15.     "max_score": 3.5410638,
  16.     "hits": [
  17.       {
  18.         "_index": ".opendistro-anomaly-detectors",
  19.         "_type": "_doc",
  20.         "_id": "m4ccEnIBTXsGi3mvMt9p",
  21.         "_version": 2,
  22.         "_seq_no": 221,
  23.         "_primary_term": 1,
  24.         "_score": 3.5410638,
  25.         "_source": {
  26.           "name": "test-detector",
  27.           "description": "Test detector",
  28.           "time_field": "timestamp",
  29.           "indices": [
  30.             "order*"
  31.           ],
  32.           "filter_query": {
  33.             "bool": {
  34.               "filter": [
  35.                 {
  36.                   "exists": {
  37.                     "field": "value",
  38.                     "boost": 1
  39.                   }
  40.                 }
  41.               ],
  42.               "adjust_pure_negative": true,
  43.               "boost": 1
  44.             }
  45.           },
  46.           "detection_interval": {
  47.             "period": {
  48.               "interval": 10,
  49.               "unit": "MINUTES"
  50.             }
  51.           },
  52.           "window_delay": {
  53.             "period": {
  54.               "interval": 1,
  55.               "unit": "MINUTES"
  56.             }
  57.           },
  58.           "schema_version": 0,
  59.           "feature_attributes": [
  60.             {
  61.               "feature_id": "xxokEnIBcpeWMD987A1X",
  62.               "feature_name": "total_order",
  63.               "feature_enabled": true,
  64.               "aggregation_query": {
  65.                 "total_order": {
  66.                   "sum": {
  67.                     "field": "value"
  68.                   }
  69.                 }
  70.               }
  71.             }
  72.           ],
  73.           "last_update_time": 1589442309241
  74.         }
  75.       }
  76.     ]
  77.   }
  78. }

Get detector stats

Provides information about how the plugin is performing.

Request

  1. GET _plugins/_anomaly_detection/stats
  2. GET _plugins/_anomaly_detection/<nodeId>/stats
  3. GET _plugins/_anomaly_detection/<nodeId>/stats/<stat>
  4. GET _plugins/_anomaly_detection/stats/<stat>

Sample response

  1. {
  2.   "_nodes" : {
  3.     "total" : 3,
  4.     "successful" : 3,
  5.     "failed" : 0
  6.   },
  7.   "cluster_name" : "multi-node-run",
  8.   "anomaly_detectors_index_status" : "green",
  9.   "detector_count" : 1,
  10.   "models_checkpoint_index_status" : "green",
  11.   "anomaly_results_index_status" : "green",
  12.   "nodes" : {
  13.     "IgWDUfzFRzW0FWAXM5FGJw" : {
  14.       "ad_execute_request_count" : 8,
  15.       "ad_execute_failure_count" : 7,
  16.       "models" : [
  17.         {
  18.           "detector_id" : "m4ccEnIBTXsGi3mvMt9p",
  19.           "model_type" : "rcf",
  20.           "model_id" : "m4ccEnIBTXsGi3mvMt9p_model_rcf_0"
  21.         },
  22.         {
  23.           "detector_id" : "m4ccEnIBTXsGi3mvMt9p",
  24.           "model_type" : "threshold",
  25.           "model_id" : "m4ccEnIBTXsGi3mvMt9p_model_threshold"
  26.         }
  27.       ]
  28.     },
  29.     "y7YUQWukQEWOYbfdEq13hQ" : {
  30.       "ad_execute_request_count" : 0,
  31.       "ad_execute_failure_count" : 0,
  32.       "models" : [ ]
  33.     },
  34.     "cDcGNsPoRAyRMlPP1m-vZw" : {
  35.       "ad_execute_request_count" : 0,
  36.       "ad_execute_failure_count" : 0,
  37.       "models" : [
  38.         {
  39.           "detector_id" : "m4ccEnIBTXsGi3mvMt9p",
  40.           "model_type" : "rcf",
  41.           "model_id" : "m4ccEnIBTXsGi3mvMt9p_model_rcf_2"
  42.         },
  43.         {
  44.           "detector_id" : "m4ccEnIBTXsGi3mvMt9p",
  45.           "model_type" : "rcf",
  46.           "model_id" : "m4ccEnIBTXsGi3mvMt9p_model_rcf_1"
  47.         }
  48.       ]
  49.     }
  50.   }
  51. }

Historical detectors contain additional fields:

Sample response

  1. {
  2.   "anomaly_detectors_index_status": "yellow",
  3.   "anomaly_detection_state_status": "yellow",
  4.   "historical_detector_count": 3,
  5.   "detector_count": 7,
  6.   "anomaly_detection_job_index_status": "yellow",
  7.   "models_checkpoint_index_status": "yellow",
  8.   "anomaly_results_index_status": "yellow",
  9.   "nodes": {
  10.     "Mz9HDZnuQwSCw0UiisxwWg": {
  11.       "ad_execute_request_count": 0,
  12.       "models": [],
  13.       "ad_canceled_batch_task_count": 2,
  14.       "ad_hc_execute_request_count": 0,
  15.       "ad_hc_execute_failure_count": 0,
  16.       "ad_execute_failure_count": 0,
  17.       "ad_batch_task_failure_count": 0,
  18.       "ad_executing_batch_task_count": 1,
  19.       "ad_total_batch_task_count": 8
  20.     }
  21.   }
  22. }

Create monitor

Create a monitor to set up alerts for the detector.

Request

  1. POST _plugins/_alerting/monitors
  2. {
  3.   "type": "monitor",
  4.   "name": "test-monitor",
  5.   "enabled": true,
  6.   "schedule": {
  7.     "period": {
  8.       "interval": 20,
  9.       "unit": "MINUTES"
  10.     }
  11.   },
  12.   "inputs": [
  13.     {
  14.       "search": {
  15.         "indices": [
  16.           ".opendistro-anomaly-results*"
  17.         ],
  18.         "query": {
  19.           "size": 1,
  20.           "query": {
  21.             "bool": {
  22.               "filter": [
  23.                 {
  24.                   "range": {
  25.                     "data_end_time": {
  26.                       "from": "||-20m",
  27.                       "to": "",
  28.                       "include_lower": true,
  29.                       "include_upper": true,
  30.                       "boost": 1
  31.                     }
  32.                   }
  33.                 },
  34.                 {
  35.                   "term": {
  36.                     "detector_id": {
  37.                       "value": "m4ccEnIBTXsGi3mvMt9p",
  38.                       "boost": 1
  39.                     }
  40.                   }
  41.                 }
  42.               ],
  43.               "adjust_pure_negative": true,
  44.               "boost": 1
  45.             }
  46.           },
  47.           "sort": [
  48.             {
  49.               "anomaly_grade": {
  50.                 "order": "desc"
  51.               }
  52.             },
  53.             {
  54.               "confidence": {
  55.                 "order": "desc"
  56.               }
  57.             }
  58.           ],
  59.           "aggregations": {
  60.             "max_anomaly_grade": {
  61.               "max": {
  62.                 "field": "anomaly_grade"
  63.               }
  64.             }
  65.           }
  66.         }
  67.       }
  68.     }
  69.   ],
  70.   "triggers": [
  71.     {
  72.       "name": "test-trigger",
  73.       "severity": "1",
  74.       "condition": {
  75.         "script": {
  76.           "source": "return ctx.results[0].aggregations.max_anomaly_grade.value != null && ctx.results[0].aggregations.max_anomaly_grade.value > 0.7 && ctx.results[0].hits.hits[0]._source.confidence > 0.7",
  77.           "lang": "painless"
  78.         }
  79.       },
  80.       "actions": [
  81.         {
  82.           "name": "test-action",
  83.           "destination_id": "ld7912sBlQ5JUWWFThoW",
  84.           "message_template": {
  85.             "source": "This is my message body."
  86.           },
  87.           "throttle_enabled": false,
  88.           "subject_template": {
  89.             "source": "TheSubject"
  90.           }
  91.         }
  92.       ]
  93.     }
  94.   ]
  95. }

Sample response

  1. {
  2.   "_id": "OClTEnIBmSf7y6LP11Jz",
  3.   "_version": 1,
  4.   "_seq_no": 10,
  5.   "_primary_term": 1,
  6.   "monitor": {
  7.     "type": "monitor",
  8.     "schema_version": 1,
  9.     "name": "test-monitor",
  10.     "enabled": true,
  11.     "enabled_time": 1589445384043,
  12.     "schedule": {
  13.       "period": {
  14.         "interval": 20,
  15.         "unit": "MINUTES"
  16.       }
  17.     },
  18.     "inputs": [
  19.       {
  20.         "search": {
  21.           "indices": [
  22.             ".opendistro-anomaly-results*"
  23.           ],
  24.           "query": {
  25.             "size": 1,
  26.             "query": {
  27.               "bool": {
  28.                 "filter": [
  29.                   {
  30.                     "range": {
  31.                       "data_end_time": {
  32.                         "from": "||-20m",
  33.                         "to": "",
  34.                         "include_lower": true,
  35.                         "include_upper": true,
  36.                         "boost": 1
  37.                       }
  38.                     }
  39.                   },
  40.                   {
  41.                     "term": {
  42.                       "detector_id": {
  43.                         "value": "m4ccEnIBTXsGi3mvMt9p",
  44.                         "boost": 1
  45.                       }
  46.                     }
  47.                   }
  48.                 ],
  49.                 "adjust_pure_negative": true,
  50.                 "boost": 1
  51.               }
  52.             },
  53.             "sort": [
  54.               {
  55.                 "anomaly_grade": {
  56.                   "order": "desc"
  57.                 }
  58.               },
  59.               {
  60.                 "confidence": {
  61.                   "order": "desc"
  62.                 }
  63.               }
  64.             ],
  65.             "aggregations": {
  66.               "max_anomaly_grade": {
  67.                 "max": {
  68.                   "field": "anomaly_grade"
  69.                 }
  70.               }
  71.             }
  72.           }
  73.         }
  74.       }
  75.     ],
  76.     "triggers": [
  77.       {
  78.         "id": "NilTEnIBmSf7y6LP11Jr",
  79.         "name": "test-trigger",
  80.         "severity": "1",
  81.         "condition": {
  82.           "script": {
  83.             "source": "return ctx.results[0].aggregations.max_anomaly_grade.value != null && ctx.results[0].aggregations.max_anomaly_grade.value > 0.7 && ctx.results[0].hits.hits[0]._source.confidence > 0.7",
  84.             "lang": "painless"
  85.           }
  86.         },
  87.         "actions": [
  88.           {
  89.             "id": "NylTEnIBmSf7y6LP11Jr",
  90.             "name": "test-action",
  91.             "destination_id": "ld7912sBlQ5JUWWFThoW",
  92.             "message_template": {
  93.               "source": "This is my message body.",
  94.               "lang": "mustache"
  95.             },
  96.             "throttle_enabled": false,
  97.             "subject_template": {
  98.               "source": "TheSubject",
  99.               "lang": "mustache"
  100.             }
  101.           }
  102.         ]
  103.       }
  104.     ],
  105.     "last_update_time": 1589445384043
  106.   }
  107. }

Profile detector

Returns information related to the current state of the detector and memory usage, including current errors and shingle size, to help troubleshoot the detector.

This command helps locate logs by identifying the nodes that run the anomaly detector job for each detector.

It also helps track the initialization percentage, the required shingles, and the estimated time left.

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/
  2. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true
  3. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/<type>
  4. GET /_plugins/_anomaly_detection/detectors/<detectorId>/_profile/<type1>,<type2>

Sample Responses

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile
  3. {
  4.     "state":"DISABLED",
  5.     "error":"Stopped detector: AD models memory usage exceeds our limit."
  6. }
  8. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true&pretty
  10. {
  11.   "state": "RUNNING",
  12.   "models": [
  13.     {
  14.       "model_id": "cneh7HEBHPICjJIdXdrR_model_rcf_2",
  15.       "model_size_in_bytes": 4456448,
  16.       "node_id": "VS29z70PSzOdHiEw4SoV9Q"
  17.     },
  18.     {
  19.       "model_id": "cneh7HEBHPICjJIdXdrR_model_rcf_1",
  20.       "model_size_in_bytes": 4456448,
  21.       "node_id": "VS29z70PSzOdHiEw4SoV9Q"
  22.     },
  23.     {
  24.       "model_id": "cneh7HEBHPICjJIdXdrR_model_threshold",
  25.       "node_id": "Og23iUroTdKrkwS-y89zLw"
  26.     },
  27.     {
  28.       "model_id": "cneh7HEBHPICjJIdXdrR_model_rcf_0",
  29.       "model_size_in_bytes": 4456448,
  30.       "node_id": "Og23iUroTdKrkwS-y89zLw"
  31.     }
  32.   ],
  33.   "shingle_size": 8,
  34.   "coordinating_node": "Og23iUroTdKrkwS-y89zLw",
  35.   "total_size_in_bytes": 13369344,
  36.   "init_progress": {
  37.     "percentage": "70%",
  38.     "estimated_minutes_left": 77,
  39.     "needed_shingles": 77
  40.   }
  41. }
  43. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/total_size_in_bytes
  45. {
  46.   "total_size_in_bytes" : 13369344
  47. }

If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory.

You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out.

Request

  1. GET /_plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true&pretty
  3. {
  4.   "state": "RUNNING",
  5.   "models": [
  6.     {
  7.       "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997684",
  8.       "model_size_in_bytes": 712480,
  9.       "node_id": "g6pmr547QR-CfpEvO67M4g"
  10.     },
  11.     {
  12.       "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997685",
  13.       "model_size_in_bytes": 712480,
  14.       "node_id": "g6pmr547QR-CfpEvO67M4g"
  15.     },
  16.     {
  17.       "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997686",
  18.       "model_size_in_bytes": 712480,
  19.       "node_id": "g6pmr547QR-CfpEvO67M4g"
  20.     },
  21.     {
  22.       "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997680",
  23.       "model_size_in_bytes": 712480,
  24.       "node_id": "g6pmr547QR-CfpEvO67M4g"
  25.     },
  26.     {
  27.       "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997681",
  28.       "model_size_in_bytes": 712480,
  29.       "node_id": "g6pmr547QR-CfpEvO67M4g"
  30.     },
  31.     {
  32.       "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997682",
  33.       "model_size_in_bytes": 712480,
  34.       "node_id": "g6pmr547QR-CfpEvO67M4g"
  35.     },
  36.     {
  37.       "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997683",
  38.       "model_size_in_bytes": 712480,
  39.       "node_id": "g6pmr547QR-CfpEvO67M4g"
  40.     }
  41.   ],
  42.   "total_size_in_bytes": 4987360,
  43.   "init_progress": {
  44.     "percentage": "100%"
  45.   },
  46.   "total_entities": 7,
  47.   "active_entities": 7
  48. }

The profile operation also provides information about each entity, such as the entity’s last_sample_timestamp and last_active_timestamp.

If there are no anomaly results for an entity, either the entity doesn’t have any sample data or its model is removed from the model cache.

last_sample_timestamp shows the last document in the input data source index containing the entity, while last_active_timestamp shows the timestamp when the entity’s model was last seen in the model cache.

Request

  1. GET /_plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true&entity=i-00f28ec1eb8997686
  2. {
  3.   "category_field": "host",
  4.   "value": "i-00f28ec1eb8997686",
  5.   "is_active": true,
  6.   "last_active_timestamp": 1604026394879,
  7.   "last_sample_timestamp": 1604026394879,
  8.   "init_progress": {
  9.     "percentage": "100%"
  10.   },
  11.   "model": {
  12.     "model_id": "TFUdd3UBBwIAGQeRh5IS_entity_i-00f28ec1eb8997686",
  13.     "model_size_in_bytes": 712480,
  14.     "node_id": "MQ-bTBW3Q2uU_2zX3pyEQg"
  15.   },
  16.   "state": "RUNNING"
  17. }

For a historical detector, specify _all or ad_task to see information about its latest task:

Request

  1. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all
  2. GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/ad_task

Sample Responses

  1. {
  2.   "ad_task": {
  3.     "ad_task": {
  4.       "task_id": "JXxyG3YBv5IHYYfMlFS2",
  5.       "last_update_time": 1606778263543,
  6.       "state": "STOPPED",
  7.       "detector_id": "SwvxCHYBPhugfWD9QAL6",
  8.       "task_progress": 0.010480972,
  9.       "init_progress": 1,
  10.       "current_piece": 1578140400000,
  11.       "execution_start_time": 1606778262709,
  12.       "is_latest": true,
  13.       "task_type": "HISTORICAL",
  14.       "detector": {
  15.         "name": "historical_test1",
  16.         "description": "test",
  17.         "time_field": "timestamp",
  18.         "indices": [
  19.           "nab_art_daily_jumpsdown"
  20.         ],
  21.         "filter_query": {
  22.           "match_all": {
  23.             "boost": 1
  24.           }
  25.         },
  26.         "detection_interval": {
  27.           "period": {
  28.             "interval": 5,
  29.             "unit": "Minutes"
  30.           }
  31.         },
  32.         "window_delay": {
  33.           "period": {
  34.             "interval": 1,
  35.             "unit": "Minutes"
  36.           }
  37.         },
  38.         "shingle_size": 8,
  39.         "schema_version": 0,
  40.         "feature_attributes": [
  41.           {
  42.             "feature_id": "zgvyCHYBPhugfWD9Ap_F",
  43.             "feature_name": "sum",
  44.             "feature_enabled": true,
  45.             "aggregation_query": {
  46.               "sum": {
  47.                 "sum": {
  48.                   "field": "value"
  49.                 }
  50.               }
  51.             }
  52.           },
  53.           {
  54.             "feature_id": "zwvyCHYBPhugfWD9Ap_G",
  55.             "feature_name": "max",
  56.             "feature_enabled": true,
  57.             "aggregation_query": {
  58.               "max": {
  59.                 "max": {
  60.                   "field": "value"
  61.                 }
  62.               }
  63.             }
  64.           }
  65.         ],
  66.         "ui_metadata": {
  67.           "features": {
  68.             "max": {
  69.               "aggregationBy": "max",
  70.               "aggregationOf": "value",
  71.               "featureType": "simple_aggs"
  72.             },
  73.             "sum": {
  74.               "aggregationBy": "sum",
  75.               "aggregationOf": "value",
  76.               "featureType": "simple_aggs"
  77.             }
  78.           },
  79.           "filters": [],
  80.           "filterType": "simple_filter"
  81.         },
  82.         "last_update_time": 1606467935713,
  83.         "detector_type": "HISTORICAL_SIGLE_ENTITY",
  84.         "detection_date_range": {
  85.           "start_time": 1577840400000,
  86.           "end_time": 1606463775000
  87.         }
  88.       }
  89.     },
  90.     "shingle_size": 8,
  91.     "rcf_total_updates": 1994,
  92.     "threshold_model_trained": true,
  93.     "threshold_model_training_data_size": 0,
  94.     "node_id": "Q9yznwxvTz-yJxtz7rJlLg"
  95.   }
  96. }