Stats API

Use the stats operation to monitor shard indexing backpressure.

Stats

Introduced 1.2

Returns node-level and shard-level stats for indexing request rejections.

Request

  1. GET _nodes/_local/stats/shard_indexing_pressure

If enforced is true:

Sample response

  1. {
  2.   "_nodes": {
  3.     "total": 1,
  4.     "successful": 1,
  5.     "failed": 0
  6.   },
  7.   "cluster_name": "runTask",
  8.   "nodes": {
  9.     "q3e1dQjFSqyPSLAgpyQlfw": {
  10.       "timestamp": 1613072111162,
  11.       "name": "runTask-0",
  12.       "transport_address": "127.0.0.1:9300",
  13.       "host": "127.0.0.1",
  14.       "ip": "127.0.0.1:9300",
  15.       "roles": [
  16.         "data",
  17.         "ingest",
  18.         "cluster_manager",
  19.         "remote_cluster_client"
  20.       ],
  21.       "attributes": {
  22.         "testattr": "test"
  23.       },
  24.       "shard_indexing_pressure": {
  25.         "stats": {
  26.           "[index_name][0]": {
  27.             "memory": {
  28.               "current": {
  29.                 "coordinating_in_bytes": 0,
  30.                 "primary_in_bytes": 0,
  31.                 "replica_in_bytes": 0
  32.               },
  33.               "total": {
  34.                 "coordinating_in_bytes": 299,
  35.                 "primary_in_bytes": 299,
  36.                 "replica_in_bytes": 0
  37.               }
  38.             },
  39.             "rejection": {
  40.               "coordinating": {
  41.                 "coordinating_rejections": 0,
  42.                 "breakup": {
  43.                   "node_limits": 0,
  44.                   "no_successful_request_limits": 0,
  45.                   "throughput_degradation_limits": 0
  46.                 }
  47.               },
  48.               "primary": {
  49.                 "primary_rejections": 0,
  50.                 "breakup": {
  51.                   "node_limits": 0,
  52.                   "no_successful_request_limits": 0,
  53.                   "throughput_degradation_limits": 0
  54.                 }
  55.               },
  56.               "replica": {
  57.                 "replica_rejections": 0,
  58.                 "breakup": {
  59.                   "node_limits": 0,
  60.                   "no_successful_request_limits": 0,
  61.                   "throughput_degradation_limits": 0
  62.                 }
  63.               }
  64.             },
  65.             "last_successful_timestamp": {
  66.               "coordinating_last_successful_request_timestamp_in_millis": 1613072107990,
  67.               "primary_last_successful_request_timestamp_in_millis": 0,
  68.               "replica_last_successful_request_timestamp_in_millis": 0
  69.             },
  70.             "indexing": {
  71.               "coordinating_time_in_millis": 96,
  72.               "coordinating_count": 1,
  73.               "primary_time_in_millis": 0,
  74.               "primary_count": 0,
  75.               "replica_time_in_millis": 0,
  76.               "replica_count": 0
  77.             },
  78.             "memory_allocation": {
  79.               "current": {
  80.                 "current_coordinating_and_primary_bytes": 0,
  81.                 "current_replica_bytes": 0
  82.               },
  83.               "limit": {
  84.                 "current_coordinating_and_primary_limits_in_bytes": 51897,
  85.                 "current_replica_limits_in_bytes": 77845
  86.               }
  87.             }
  88.           }
  89.         },
  90.         "total_rejections_breakup": {
  91.           "node_limits": 0,
  92.           "no_successful_request_limits": 0,
  93.           "throughput_degradation_limits": 0
  94.         },
  95.         "enabled": true,
  96.         "enforced" : true
  97.       }
  98.     }
  99.   }
  100. }

If enforced is false:

Sample response

  1. {
  2.   "_nodes": {
  3.     "total": 1,
  4.     "successful": 1,
  5.     "failed": 0
  6.   },
  7.   "cluster_name": "runTask",
  8.   "nodes": {
  9.     "q3e1dQjFSqyPSLAgpyQlfw": {
  10.       "timestamp": 1613072111162,
  11.       "name": "runTask-0",
  12.       "transport_address": "127.0.0.1:9300",
  13.       "host": "127.0.0.1",
  14.       "ip": "127.0.0.1:9300",
  15.       "roles": [
  16.         "data",
  17.         "ingest",
  18.         "cluster_manager",
  19.         "remote_cluster_client"
  20.       ],
  21.       "attributes": {
  22.         "testattr": "test"
  23.       },
  24.       "shard_indexing_pressure": {
  25.         "stats": {
  26.           "[index_name][0]": {
  27.             "memory": {
  28.               "current": {
  29.                 "coordinating_in_bytes": 0,
  30.                 "primary_in_bytes": 0,
  31.                 "replica_in_bytes": 0
  32.               },
  33.               "total": {
  34.                 "coordinating_in_bytes": 299,
  35.                 "primary_in_bytes": 299,
  36.                 "replica_in_bytes": 0
  37.               }
  38.             },
  39.             "rejection": {
  40.               "coordinating": {
  41.                 "coordinating_rejections": 0,
  42.                 "breakup_shadow_mode": {
  43.                   "node_limits": 0,
  44.                   "no_successful_request_limits": 0,
  45.                   "throughput_degradation_limits": 0
  46.                 }
  47.               },
  48.               "primary": {
  49.                 "primary_rejections": 0,
  50.                 "breakup_shadow_mode": {
  51.                   "node_limits": 0,
  52.                   "no_successful_request_limits": 0,
  53.                   "throughput_degradation_limits": 0
  54.                 }
  55.               },
  56.               "replica": {
  57.                 "replica_rejections": 0,
  58.                 "breakup_shadow_mode": {
  59.                   "node_limits": 0,
  60.                   "no_successful_request_limits": 0,
  61.                   "throughput_degradation_limits": 0
  62.                 }
  63.               }
  64.             },
  65.             "last_successful_timestamp": {
  66.               "coordinating_last_successful_request_timestamp_in_millis": 1613072107990,
  67.               "primary_last_successful_request_timestamp_in_millis": 0,
  68.               "replica_last_successful_request_timestamp_in_millis": 0
  69.             },
  70.             "indexing": {
  71.               "coordinating_time_in_millis": 96,
  72.               "coordinating_count": 1,
  73.               "primary_time_in_millis": 0,
  74.               "primary_count": 0,
  75.               "replica_time_in_millis": 0,
  76.               "replica_count": 0
  77.             },
  78.             "memory_allocation": {
  79.               "current": {
  80.                 "current_coordinating_and_primary_bytes": 0,
  81.                 "current_replica_bytes": 0
  82.               },
  83.               "limit": {
  84.                 "current_coordinating_and_primary_limits_in_bytes": 51897,
  85.                 "current_replica_limits_in_bytes": 77845
  86.               }
  87.             }
  88.           }
  89.         },
  90.         "total_rejections_breakup_shadow_mode": {
  91.           "node_limits": 0,
  92.           "no_successful_request_limits": 0,
  93.           "throughput_degradation_limits": 0
  94.         },
  95.         "enabled": true,
  96.         "enforced" : false
  97.       }
  98.     }
  99.   }
  100. }

To include all the shards with both active and previous write operations performed on them, specify the include_all parameter:

Request

  1. GET _nodes/_local/stats/shard_indexing_pressure?include_all

Sample response

  1. {
  2.   "_nodes": {
  3.     "total": 1,
  4.     "successful": 1,
  5.     "failed": 0
  6.   },
  7.   "cluster_name": "runTask",
  8.   "nodes": {
  9.     "q3e1dQjFSqyPSLAgpyQlfw": {
  10.       "timestamp": 1613072198171,
  11.       "name": "runTask-0",
  12.       "transport_address": "127.0.0.1:9300",
  13.       "host": "127.0.0.1",
  14.       "ip": "127.0.0.1:9300",
  15.       "roles": [
  16.         "data",
  17.         "ingest",
  18.         "cluster_manager",
  19.         "remote_cluster_client"
  20.       ],
  21.       "attributes": {
  22.         "testattr": "test"
  23.       },
  24.       "shard_indexing_pressure": {
  25.         "stats": {
  26.           "[index_name][0]": {
  27.             "memory": {
  28.               "current": {
  29.                 "coordinating_in_bytes": 0,
  30.                 "primary_in_bytes": 0,
  31.                 "replica_in_bytes": 0
  32.               },
  33.               "total": {
  34.                 "coordinating_in_bytes": 604,
  35.                 "primary_in_bytes": 604,
  36.                 "replica_in_bytes": 0
  37.               }
  38.             },
  39.             "rejection": {
  40.               "coordinating": {
  41.                 "coordinating_rejections": 0,
  42.                 "breakup": {
  43.                   "node_limits": 0,
  44.                   "no_successful_request_limits": 0,
  45.                   "throughput_degradation_limits": 0
  46.                 }
  47.               },
  48.               "primary": {
  49.                 "primary_rejections": 0,
  50.                 "breakup": {
  51.                   "node_limits": 0,
  52.                   "no_successful_request_limits": 0,
  53.                   "throughput_degradation_limits": 0
  54.                 }
  55.               },
  56.               "replica": {
  57.                 "replica_rejections": 0,
  58.                 "breakup": {
  59.                   "node_limits": 0,
  60.                   "no_successful_request_limits": 0,
  61.                   "throughput_degradation_limits": 0
  62.                 }
  63.               }
  64.             },
  65.             "last_successful_timestamp": {
  66.               "coordinating_last_successful_request_timestamp_in_millis": 1613072194656,
  67.               "primary_last_successful_request_timestamp_in_millis": 0,
  68.               "replica_last_successful_request_timestamp_in_millis": 0
  69.             },
  70.             "indexing": {
  71.               "coordinating_time_in_millis": 145,
  72.               "coordinating_count": 2,
  73.               "primary_time_in_millis": 0,
  74.               "primary_count": 0,
  75.               "replica_time_in_millis": 0,
  76.               "replica_count": 0
  77.             },
  78.             "memory_allocation": {
  79.               "current": {
  80.                 "current_coordinating_and_primary_bytes": 0,
  81.                 "current_replica_bytes": 0
  82.               },
  83.               "limit": {
  84.                 "current_coordinating_and_primary_limits_in_bytes": 51897,
  85.                 "current_replica_limits_in_bytes": 77845
  86.               }
  87.             }
  88.           }
  89.         },
  90.         "total_rejections_breakup": {
  91.           "node_limits": 0,
  92.           "no_successful_request_limits": 0,
  93.           "throughput_degradation_limits": 0
  94.         },
  95.         "enabled": true,
  96.         "enforced": true
  97.       }
  98.     }
  99.   }
  100. }

To get only all the top-level aggregated stats, specify the top parameter (skips the per-shard stats).

Request

  1. GET _nodes/_local/stats/shard_indexing_pressure?top

If enforced is true:

Sample response

  1. {
  2.   "_nodes": {
  3.     "total": 1,
  4.     "successful": 1,
  5.     "failed": 0
  6.   },
  7.   "cluster_name": "runTask",
  8.   "nodes": {
  9.     "q3e1dQjFSqyPSLAgpyQlfw": {
  10.       "timestamp": 1613072382719,
  11.       "name": "runTask-0",
  12.       "transport_address": "127.0.0.1:9300",
  13.       "host": "127.0.0.1",
  14.       "ip": "127.0.0.1:9300",
  15.       "roles": [
  16.         "data",
  17.         "ingest",
  18.         "cluster_manager",
  19.         "remote_cluster_client"
  20.       ],
  21.       "attributes": {
  22.         "testattr": "test"
  23.       },
  24.       "shard_indexing_pressure": {
  25.         "stats": {},
  26.         "total_rejections_breakup": {
  27.           "node_limits": 0,
  28.           "no_successful_request_limits": 0,
  29.           "throughput_degradation_limits": 0
  30.         },
  31.         "enabled": true,
  32.         "enforced": true
  33.       }
  34.     }
  35.   }
  36. }

If enforced is false:

Sample response

  1. {
  2.   "_nodes": {
  3.     "total": 1,
  4.     "successful": 1,
  5.     "failed": 0
  6.   },
  7.   "cluster_name": "runTask",
  8.   "nodes": {
  9.     "q3e1dQjFSqyPSLAgpyQlfw": {
  10.       "timestamp": 1613072382719,
  11.       "name": "runTask-0",
  12.       "transport_address": "127.0.0.1:9300",
  13.       "host": "127.0.0.1",
  14.       "ip": "127.0.0.1:9300",
  15.       "roles": [
  16.         "data",
  17.         "ingest",
  18.         "cluster_manager",
  19.         "remote_cluster_client"
  20.       ],
  21.       "attributes": {
  22.         "testattr": "test"
  23.       },
  24.       "shard_indexing_pressure": {
  25.         "stats": {},
  26.         "total_rejections_breakup_shadow_mode": {
  27.           "node_limits": 0,
  28.           "no_successful_request_limits": 0,
  29.           "throughput_degradation_limits": 0
  30.         },
  31.         "enabled": true,
  32.         "enforced" : false
  33.       }
  34.     }
  35.   }
  36. }

To get the shard-level breakup of rejections for every node (only includes shards with active write operations):

Request

  1. GET _nodes/stats/shard_indexing_pressure

Sample response

  1. {
  2.   "_nodes": {
  3.     "total": 1,
  4.     "successful": 1,
  5.     "failed": 0
  6.   },
  7.   "cluster_name": "runTask",
  8.   "nodes": {
  9.     "q3e1dQjFSqyPSLAgpyQlfw": {
  10.       "timestamp": 1613072382719,
  11.       "name": "runTask-0",
  12.       "transport_address": "127.0.0.1:9300",
  13.       "host": "127.0.0.1",
  14.       "ip": "127.0.0.1:9300",
  15.       "roles": [
  16.         "data",
  17.         "ingest",
  18.         "cluster_manager",
  19.         "remote_cluster_client"
  20.       ],
  21.       "attributes": {
  22.         "testattr": "test"
  23.       },
  24.       "shard_indexing_pressure": {
  25.         "stats": {},
  26.         "total_rejections_breakup": {
  27.           "node_limits": 0,
  28.           "no_successful_request_limits": 0,
  29.           "throughput_degradation_limits": 0
  30.         },
  31.         "enabled": true,
  32.         "enforced": true
  33.       }
  34.     }
  35.   }
  36. }