Stats API

Use the stats operation to monitor shard indexing backpressure.

Stats

Introduced 1.2

Returns node-level and shard-level stats for indexing request rejections.

Request

  1. GET _nodes/_local/stats/shard_indexing_pressure

If enforced is true:

Sample response

  1. {
  2. "_nodes": {
  3. "total": 1,
  4. "successful": 1,
  5. "failed": 0
  6. },
  7. "cluster_name": "runTask",
  8. "nodes": {
  9. "q3e1dQjFSqyPSLAgpyQlfw": {
  10. "timestamp": 1613072111162,
  11. "name": "runTask-0",
  12. "transport_address": "127.0.0.1:9300",
  13. "host": "127.0.0.1",
  14. "ip": "127.0.0.1:9300",
  15. "roles": [
  16. "data",
  17. "ingest",
  18. "master",
  19. "remote_cluster_client"
  20. ],
  21. "attributes": {
  22. "testattr": "test"
  23. },
  24. "shard_indexing_pressure": {
  25. "stats": {
  26. "[index_name][0]": {
  27. "memory": {
  28. "current": {
  29. "coordinating_in_bytes": 0,
  30. "primary_in_bytes": 0,
  31. "replica_in_bytes": 0
  32. },
  33. "total": {
  34. "coordinating_in_bytes": 299,
  35. "primary_in_bytes": 299,
  36. "replica_in_bytes": 0
  37. }
  38. },
  39. "rejection": {
  40. "coordinating": {
  41. "coordinating_rejections": 0,
  42. "breakup": {
  43. "node_limits": 0,
  44. "no_successful_request_limits": 0,
  45. "throughput_degradation_limits": 0
  46. }
  47. },
  48. "primary": {
  49. "primary_rejections": 0,
  50. "breakup": {
  51. "node_limits": 0,
  52. "no_successful_request_limits": 0,
  53. "throughput_degradation_limits": 0
  54. }
  55. },
  56. "replica": {
  57. "replica_rejections": 0,
  58. "breakup": {
  59. "node_limits": 0,
  60. "no_successful_request_limits": 0,
  61. "throughput_degradation_limits": 0
  62. }
  63. }
  64. },
  65. "last_successful_timestamp": {
  66. "coordinating_last_successful_request_timestamp_in_millis": 1613072107990,
  67. "primary_last_successful_request_timestamp_in_millis": 0,
  68. "replica_last_successful_request_timestamp_in_millis": 0
  69. },
  70. "indexing": {
  71. "coordinating_time_in_millis": 96,
  72. "coordinating_count": 1,
  73. "primary_time_in_millis": 0,
  74. "primary_count": 0,
  75. "replica_time_in_millis": 0,
  76. "replica_count": 0
  77. },
  78. "memory_allocation": {
  79. "current": {
  80. "current_coordinating_and_primary_bytes": 0,
  81. "current_replica_bytes": 0
  82. },
  83. "limit": {
  84. "current_coordinating_and_primary_limits_in_bytes": 51897,
  85. "current_replica_limits_in_bytes": 77845
  86. }
  87. }
  88. }
  89. },
  90. "total_rejections_breakup": {
  91. "node_limits": 0,
  92. "no_successful_request_limits": 0,
  93. "throughput_degradation_limits": 0
  94. },
  95. "enabled": true,
  96. "enforced" : true
  97. }
  98. }
  99. }
  100. }

If enforced is false:

Sample response

  1. {
  2. "_nodes": {
  3. "total": 1,
  4. "successful": 1,
  5. "failed": 0
  6. },
  7. "cluster_name": "runTask",
  8. "nodes": {
  9. "q3e1dQjFSqyPSLAgpyQlfw": {
  10. "timestamp": 1613072111162,
  11. "name": "runTask-0",
  12. "transport_address": "127.0.0.1:9300",
  13. "host": "127.0.0.1",
  14. "ip": "127.0.0.1:9300",
  15. "roles": [
  16. "data",
  17. "ingest",
  18. "master",
  19. "remote_cluster_client"
  20. ],
  21. "attributes": {
  22. "testattr": "test"
  23. },
  24. "shard_indexing_pressure": {
  25. "stats": {
  26. "[index_name][0]": {
  27. "memory": {
  28. "current": {
  29. "coordinating_in_bytes": 0,
  30. "primary_in_bytes": 0,
  31. "replica_in_bytes": 0
  32. },
  33. "total": {
  34. "coordinating_in_bytes": 299,
  35. "primary_in_bytes": 299,
  36. "replica_in_bytes": 0
  37. }
  38. },
  39. "rejection": {
  40. "coordinating": {
  41. "coordinating_rejections": 0,
  42. "breakup_shadow_mode": {
  43. "node_limits": 0,
  44. "no_successful_request_limits": 0,
  45. "throughput_degradation_limits": 0
  46. }
  47. },
  48. "primary": {
  49. "primary_rejections": 0,
  50. "breakup_shadow_mode": {
  51. "node_limits": 0,
  52. "no_successful_request_limits": 0,
  53. "throughput_degradation_limits": 0
  54. }
  55. },
  56. "replica": {
  57. "replica_rejections": 0,
  58. "breakup_shadow_mode": {
  59. "node_limits": 0,
  60. "no_successful_request_limits": 0,
  61. "throughput_degradation_limits": 0
  62. }
  63. }
  64. },
  65. "last_successful_timestamp": {
  66. "coordinating_last_successful_request_timestamp_in_millis": 1613072107990,
  67. "primary_last_successful_request_timestamp_in_millis": 0,
  68. "replica_last_successful_request_timestamp_in_millis": 0
  69. },
  70. "indexing": {
  71. "coordinating_time_in_millis": 96,
  72. "coordinating_count": 1,
  73. "primary_time_in_millis": 0,
  74. "primary_count": 0,
  75. "replica_time_in_millis": 0,
  76. "replica_count": 0
  77. },
  78. "memory_allocation": {
  79. "current": {
  80. "current_coordinating_and_primary_bytes": 0,
  81. "current_replica_bytes": 0
  82. },
  83. "limit": {
  84. "current_coordinating_and_primary_limits_in_bytes": 51897,
  85. "current_replica_limits_in_bytes": 77845
  86. }
  87. }
  88. }
  89. },
  90. "total_rejections_breakup_shadow_mode": {
  91. "node_limits": 0,
  92. "no_successful_request_limits": 0,
  93. "throughput_degradation_limits": 0
  94. },
  95. "enabled": true,
  96. "enforced" : false
  97. }
  98. }
  99. }
  100. }

To include all the shards with both active and previous write operations performed on them, specify the include_all parameter:

Request

  1. GET _nodes/_local/stats/shard_indexing_pressure?include_all

Sample response

  1. {
  2. "_nodes": {
  3. "total": 1,
  4. "successful": 1,
  5. "failed": 0
  6. },
  7. "cluster_name": "runTask",
  8. "nodes": {
  9. "q3e1dQjFSqyPSLAgpyQlfw": {
  10. "timestamp": 1613072198171,
  11. "name": "runTask-0",
  12. "transport_address": "127.0.0.1:9300",
  13. "host": "127.0.0.1",
  14. "ip": "127.0.0.1:9300",
  15. "roles": [
  16. "data",
  17. "ingest",
  18. "master",
  19. "remote_cluster_client"
  20. ],
  21. "attributes": {
  22. "testattr": "test"
  23. },
  24. "shard_indexing_pressure": {
  25. "stats": {
  26. "[index_name][0]": {
  27. "memory": {
  28. "current": {
  29. "coordinating_in_bytes": 0,
  30. "primary_in_bytes": 0,
  31. "replica_in_bytes": 0
  32. },
  33. "total": {
  34. "coordinating_in_bytes": 604,
  35. "primary_in_bytes": 604,
  36. "replica_in_bytes": 0
  37. }
  38. },
  39. "rejection": {
  40. "coordinating": {
  41. "coordinating_rejections": 0,
  42. "breakup": {
  43. "node_limits": 0,
  44. "no_successful_request_limits": 0,
  45. "throughput_degradation_limits": 0
  46. }
  47. },
  48. "primary": {
  49. "primary_rejections": 0,
  50. "breakup": {
  51. "node_limits": 0,
  52. "no_successful_request_limits": 0,
  53. "throughput_degradation_limits": 0
  54. }
  55. },
  56. "replica": {
  57. "replica_rejections": 0,
  58. "breakup": {
  59. "node_limits": 0,
  60. "no_successful_request_limits": 0,
  61. "throughput_degradation_limits": 0
  62. }
  63. }
  64. },
  65. "last_successful_timestamp": {
  66. "coordinating_last_successful_request_timestamp_in_millis": 1613072194656,
  67. "primary_last_successful_request_timestamp_in_millis": 0,
  68. "replica_last_successful_request_timestamp_in_millis": 0
  69. },
  70. "indexing": {
  71. "coordinating_time_in_millis": 145,
  72. "coordinating_count": 2,
  73. "primary_time_in_millis": 0,
  74. "primary_count": 0,
  75. "replica_time_in_millis": 0,
  76. "replica_count": 0
  77. },
  78. "memory_allocation": {
  79. "current": {
  80. "current_coordinating_and_primary_bytes": 0,
  81. "current_replica_bytes": 0
  82. },
  83. "limit": {
  84. "current_coordinating_and_primary_limits_in_bytes": 51897,
  85. "current_replica_limits_in_bytes": 77845
  86. }
  87. }
  88. }
  89. },
  90. "total_rejections_breakup": {
  91. "node_limits": 0,
  92. "no_successful_request_limits": 0,
  93. "throughput_degradation_limits": 0
  94. },
  95. "enabled": true,
  96. "enforced": true
  97. }
  98. }
  99. }
  100. }

To get only all the top-level aggregated stats, specify the top parameter (skips the per-shard stats).

Request

  1. GET _nodes/_local/stats/shard_indexing_pressure?top

If enforced is true:

Sample response

  1. {
  2. "_nodes": {
  3. "total": 1,
  4. "successful": 1,
  5. "failed": 0
  6. },
  7. "cluster_name": "runTask",
  8. "nodes": {
  9. "q3e1dQjFSqyPSLAgpyQlfw": {
  10. "timestamp": 1613072382719,
  11. "name": "runTask-0",
  12. "transport_address": "127.0.0.1:9300",
  13. "host": "127.0.0.1",
  14. "ip": "127.0.0.1:9300",
  15. "roles": [
  16. "data",
  17. "ingest",
  18. "master",
  19. "remote_cluster_client"
  20. ],
  21. "attributes": {
  22. "testattr": "test"
  23. },
  24. "shard_indexing_pressure": {
  25. "stats": {},
  26. "total_rejections_breakup": {
  27. "node_limits": 0,
  28. "no_successful_request_limits": 0,
  29. "throughput_degradation_limits": 0
  30. },
  31. "enabled": true,
  32. "enforced": true
  33. }
  34. }
  35. }
  36. }

If enforced is false:

Sample response

  1. {
  2. "_nodes": {
  3. "total": 1,
  4. "successful": 1,
  5. "failed": 0
  6. },
  7. "cluster_name": "runTask",
  8. "nodes": {
  9. "q3e1dQjFSqyPSLAgpyQlfw": {
  10. "timestamp": 1613072382719,
  11. "name": "runTask-0",
  12. "transport_address": "127.0.0.1:9300",
  13. "host": "127.0.0.1",
  14. "ip": "127.0.0.1:9300",
  15. "roles": [
  16. "data",
  17. "ingest",
  18. "master",
  19. "remote_cluster_client"
  20. ],
  21. "attributes": {
  22. "testattr": "test"
  23. },
  24. "shard_indexing_pressure": {
  25. "stats": {},
  26. "total_rejections_breakup_shadow_mode": {
  27. "node_limits": 0,
  28. "no_successful_request_limits": 0,
  29. "throughput_degradation_limits": 0
  30. },
  31. "enabled": true,
  32. "enforced" : false
  33. }
  34. }
  35. }
  36. }

To get the shard-level breakup of rejections for every node (only includes shards with active write operations):

Request

  1. GET _nodes/stats/shard_indexing_pressure

Sample response

  1. {
  2. "_nodes": {
  3. "total": 1,
  4. "successful": 1,
  5. "failed": 0
  6. },
  7. "cluster_name": "runTask",
  8. "nodes": {
  9. "q3e1dQjFSqyPSLAgpyQlfw": {
  10. "timestamp": 1613072382719,
  11. "name": "runTask-0",
  12. "transport_address": "127.0.0.1:9300",
  13. "host": "127.0.0.1",
  14. "ip": "127.0.0.1:9300",
  15. "roles": [
  16. "data",
  17. "ingest",
  18. "master",
  19. "remote_cluster_client"
  20. ],
  21. "attributes": {
  22. "testattr": "test"
  23. },
  24. "shard_indexing_pressure": {
  25. "stats": {},
  26. "total_rejections_breakup": {
  27. "node_limits": 0,
  28. "no_successful_request_limits": 0,
  29. "throughput_degradation_limits": 0
  30. },
  31. "enabled": true,
  32. "enforced": true
  33. }
  34. }
  35. }
  36. }