Remote Store Stats API

Introduced 2.8

Use the Remote Store Stats API to monitor shard-level remote-backed storage performance.

Metrics returned from this API only relate to indexes stored on remote-backed nodes. For an aggregated output on an index at the node or cluster level, use the Index Stats, Nodes Stats, or Cluster Stats API.

Path and HTTP methods

  1. GET _remotestore/stats/<index_name>
  2. GET _remotestore/stats/<index_name>/<shard_id>

Path parameters

The following table lists the available path parameters. All path parameters are optional.

ParameterTypeDescription
index_nameStringThe index name or index pattern.
shard_idStringThe shard ID.

Remote store stats for an index

Use the following API to get remote store statistics for all index shards.

Example request

  1. GET _remotestore/stats/<index_name>

copy

Example response

Response

  1. {
  2.     "_shards": {
  3.         "total": 4,
  4.         "successful": 4,
  5.         "failed": 0
  6.     },
  7.     "indices": {
  8.         "remote-index": {
  9.             "shards": {
  10.                 "0": [{
  11.                         "routing": {
  12.                             "state": "STARTED",
  13.                             "primary": true,
  14.                             "node": "q1VxWZnCTICrfRc2bRW3nw"
  15.                         },
  16.                         "segment": {
  17.                             "download": {},
  18.                             "upload": {
  19.                                 "local_refresh_timestamp_in_millis": 1694171634102,
  20.                                 "remote_refresh_timestamp_in_millis": 1694171634102,
  21.                                 "refresh_time_lag_in_millis": 0,
  22.                                 "refresh_lag": 0,
  23.                                 "bytes_lag": 0,
  24.                                 "backpressure_rejection_count": 0,
  25.                                 "consecutive_failure_count": 0,
  26.                                 "total_uploads": {
  27.                                     "started": 5,
  28.                                     "succeeded": 5,
  29.                                     "failed": 0
  30.                                 },
  31.                                 "total_upload_size": {
  32.                                     "started_bytes": 15342,
  33.                                     "succeeded_bytes": 15342,
  34.                                     "failed_bytes": 0
  35.                                 },
  36.                                 "remote_refresh_size_in_bytes": {
  37.                                     "last_successful": 0,
  38.                                     "moving_avg": 3068.4
  39.                                 },
  40.                                 "upload_speed_in_bytes_per_sec": {
  41.                                     "moving_avg": 99988.2
  42.                                 },
  43.                                 "remote_refresh_latency_in_millis": {
  44.                                     "moving_avg": 44.0
  45.                                 }
  46.                             }
  47.                         },
  48.                         "translog": {
  49.                             "upload": {
  50.                                 "last_successful_upload_timestamp": 1694171633644,
  51.                                 "total_uploads": {
  52.                                     "started": 6,
  53.                                     "failed": 0,
  54.                                     "succeeded": 6
  55.                                 },
  56.                                 "total_upload_size": {
  57.                                     "started_bytes": 1932,
  58.                                     "failed_bytes": 0,
  59.                                     "succeeded_bytes": 1932
  60.                                 },
  61.                                 "total_upload_time_in_millis": 21478,
  62.                                 "upload_size_in_bytes": {
  63.                                     "moving_avg": 322.0
  64.                                 },
  65.                                 "upload_speed_in_bytes_per_sec": {
  66.                                     "moving_avg": 2073.8333333333335
  67.                                 },
  68.                                 "upload_time_in_millis": {
  69.                                     "moving_avg": 3579.6666666666665
  70.                                 }
  71.                             },
  72.                             "download": {}
  73.                         }
  74.                     },
  75.                     {
  76.                         "routing": {
  77.                             "state": "STARTED",
  78.                             "primary": false,
  79.                             "node": "EZuen5Y5Sv-eDCLwh9gv-Q"
  80.                         },
  81.                         "segment": {
  82.                             "download": {
  83.                                 "last_sync_timestamp": 1694171634148,
  84.                                 "total_download_size": {
  85.                                     "started_bytes": 15112,
  86.                                     "succeeded_bytes": 15112,
  87.                                     "failed_bytes": 0
  88.                                 },
  89.                                 "download_size_in_bytes": {
  90.                                     "last_successful": 2910,
  91.                                     "moving_avg": 1259.3333333333333
  92.                                 },
  93.                                 "download_speed_in_bytes_per_sec": {
  94.                                     "moving_avg": 382387.3333333333
  95.                                 }
  96.                             },
  97.                             "upload": {}
  98.                         },
  99.                         "translog": {
  100.                             "upload": {},
  101.                             "download": {}
  102.                         }
  103.                     }
  104.                 ],
  105.                 "1": [{
  106.                         "routing": {
  107.                             "state": "STARTED",
  108.                             "primary": false,
  109.                             "node": "q1VxWZnCTICrfRc2bRW3nw"
  110.                         },
  111.                         "segment": {
  112.                             "download": {
  113.                                 "last_sync_timestamp": 1694171633181,
  114.                                 "total_download_size": {
  115.                                     "started_bytes": 18978,
  116.                                     "succeeded_bytes": 18978,
  117.                                     "failed_bytes": 0
  118.                                 },
  119.                                 "download_size_in_bytes": {
  120.                                     "last_successful": 325,
  121.                                     "moving_avg": 1265.2
  122.                                 },
  123.                                 "download_speed_in_bytes_per_sec": {
  124.                                     "moving_avg": 456047.6666666667
  125.                                 }
  126.                             },
  127.                             "upload": {}
  128.                         },
  129.                         "translog": {
  130.                             "upload": {},
  131.                             "download": {}
  132.                         }
  133.                     },
  134.                     {
  135.                         "routing": {
  136.                             "state": "STARTED",
  137.                             "primary": true,
  138.                             "node": "EZuen5Y5Sv-eDCLwh9gv-Q"
  139.                         },
  140.                         "segment": {
  141.                             "download": {},
  142.                             "upload": {
  143.                                 "local_refresh_timestamp_in_millis": 1694171633122,
  144.                                 "remote_refresh_timestamp_in_millis": 1694171633122,
  145.                                 "refresh_time_lag_in_millis": 0,
  146.                                 "refresh_lag": 0,
  147.                                 "bytes_lag": 0,
  148.                                 "backpressure_rejection_count": 0,
  149.                                 "consecutive_failure_count": 0,
  150.                                 "total_uploads": {
  151.                                     "started": 6,
  152.                                     "succeeded": 6,
  153.                                     "failed": 0
  154.                                 },
  155.                                 "total_upload_size": {
  156.                                     "started_bytes": 19208,
  157.                                     "succeeded_bytes": 19208,
  158.                                     "failed_bytes": 0
  159.                                 },
  160.                                 "remote_refresh_size_in_bytes": {
  161.                                     "last_successful": 0,
  162.                                     "moving_avg": 3201.3333333333335
  163.                                 },
  164.                                 "upload_speed_in_bytes_per_sec": {
  165.                                     "moving_avg": 109612.0
  166.                                 },
  167.                                 "remote_refresh_latency_in_millis": {
  168.                                     "moving_avg": 25.333333333333332
  169.                                 }
  170.                             }
  171.                         },
  172.                         "translog": {
  173.                             "upload": {
  174.                                 "last_successful_upload_timestamp": 1694171633106,
  175.                                 "total_uploads": {
  176.                                     "started": 7,
  177.                                     "failed": 0,
  178.                                     "succeeded": 7
  179.                                 },
  180.                                 "total_upload_size": {
  181.                                     "started_bytes": 2405,
  182.                                     "failed_bytes": 0,
  183.                                     "succeeded_bytes": 2405
  184.                                 },
  185.                                 "total_upload_time_in_millis": 27748,
  186.                                 "upload_size_in_bytes": {
  187.                                     "moving_avg": 343.57142857142856
  188.                                 },
  189.                                 "upload_speed_in_bytes_per_sec": {
  190.                                     "moving_avg": 1445.857142857143
  191.                                 },
  192.                                 "upload_time_in_millis": {
  193.                                     "moving_avg": 3964.0
  194.                                 }
  195.                             },
  196.                             "download": {}
  197.                         }
  198.                     }
  199.                 ]
  200.             }
  201.         }
  202.     }
  203. }

Response body fields

The response body of the Remote Store Stats API is split into three categories:

  • routing : Contains information related to the shard’s routing
  • segment : Contains statistics related to segment transfers from remote-backed storage
  • translog : Contains statistics related to translog transfers from remote-backed storage

routing

The routing object contains the following fields.

FieldDescription
primaryDenotes whether the shard copy is a primary shard.
nodeThe name of the node to which the shard is assigned.

segment

The segment.upload object contains the following fields.

FieldDescription
local_refresh_timestamp_in_millisThe last successful local refresh timestamp, in milliseconds.
remote_refresh_timestamp_in_millisThe last successful remote refresh timestamp, in milliseconds.
refresh_time_lag_in_millisThe amount of time, in milliseconds, that the remote refresh is behind the local refresh.
refresh_lagThe number of refreshes by which the remote store is lagging behind the local store.
bytes_lagThe lag, in bytes, between the remote and local stores.
backpressure_rejection_countThe total number of write rejections issued due to backpressure in the remote store.
consecutive_failure_countThe number of consecutive remote refresh failures since the last successful refresh.
total_remote_refreshThe total number of remote refreshes.
total_uploads_in_bytesThe total number of bytes in all uploads to the remote store.
remote_refresh_size_in_bytes.last_successfulThe size of the data uploaded during the last successful refresh.
remote_refresh_size_in_bytes.moving_avgThe average size of the data, in bytes, uploaded in the last N refreshes. N is defined in the remote_store.moving_average_window_size setting. For more information, see Remote segment backpressure.
upload_latency_in_bytes_per_sec.moving_avgThe average speed of remote segment uploads, in bytes per second, for the last N uploads. N is defined in the remote_store.moving_average_window_size setting. For more information, see Remote segment backpressure.
remote_refresh_latency_in_millis.moving_avgThe average amount of time, in milliseconds, taken by a single remote refresh during the last N remote refreshes. N is defined in the remote_store.moving_average_window_size setting. For more information, see Remote segment backpressure.

The segment.download object contains the following fields.

FieldDescription
last_sync_timestampThe timestamp, in milliseconds, since the last successful segment file download from remote-backed storage.
total_download_size.started_bytesThe total number of bytes of segment files actively being downloaded from remote-backed storage.
total_download_size.succeeded_bytesThe total number of bytes of segment files successfully downloaded from remote-backed storage.
total_download_size.failed_bytesThe total number of bytes of segment files that failed to download from remote-back storage.
download_size_in_bytes.last_successfulThe size, in bytes, of the last segment file successfully downloaded from remote-backed storage.
download_size_in_bytes.moving_avgThe average size of segment data, in bytes, downloaded in the last 20 downloads.
download_speed_in_bytes_per_sec.moving_avgThe average download speed, in bytes per second, of the last 20 downloads.

translog

The translog.upload object contains the following fields.

FieldDescription
last_successful_upload_timestampThe timestamp, in milliseconds, since the last translog file successfully uploaded to remote-backed storage.
total_uploads.startedThe total number of attempted translog upload syncs to remote-backed storage.
total_uploads.failedThe total number of failed translog upload syncs to remote-backed storage.
total_uploads.succeededThe total number of successful translog upload syncs to remote-backed storage.
total_upload_size.started_bytesThe total number of bytes of translog files actively being downloaded from remote-backed storage.
total_upload_size.succeeded_bytesThe total number of bytes of translog files successfully uploaded to remote-backed storage.
total_upload_size.failed_bytesThe total number of bytes of translog files that failed to upload to remote-backed storage.
total_upload_time_in_millisThe total amount of time spent, in milliseconds, uploading translog files to remote-backed storage.
upload_size_in_bytes.moving_avgThe average size of translog data, in bytes, uploaded in the last N downloads. N is defined in the remote_store.moving_average_window_size setting.
upload_speed_in_bytes_per_sec.moving_avgThe average speed of translog uploads, in bytes per second, for the last N uploads. N is defined in the remote_store.moving_average_window_size setting.
upload_time_in_millis.moving_avgThe average amount of time taken by a single translog upload, in milliseconds, since the last N uploads. N is defined in the remote_store.moving_average_window_size setting.

The translog.download object contains the following fields.

FieldDescription
last_successful_download_timestampThe timestamp, in milliseconds, since the last translog file successfully uploaded to remote-backed storage.
total_downloads.succeededThe total number of successful translog download syncs from remote-backed storage.
total_download_size.succeeded_bytesThe total number of bytes of translog files successfully uploaded from remote-backed storage.
total_download_time_in_millisThe total amount of time spent, in milliseconds, downloading translog files from remote-backed storage.
download_size_in_bytes.moving_avgThe average size of translog data, in bytes, downloaded in the last N downloads. N is defined in the remote_store.moving_average_window_size setting.
download_speed_in_bytes_per_sec.moving_avgThe average speed of translog downloads, in bytes per second, for the last N uploads. N is defined in the remote_store.moving_average_window_size setting.
download_time_in_millis.moving_avgThe average amount of time taken by a single translog download, in milliseconds, since the last N uploads. N is defined in the remote_store.moving_average_window_size setting.

Remote store stats for a single shard

Use the following API to get remote store statistics for a single shard.

Example request

  1. GET _remotestore/stats/<index_name>/<shard_id>

copy

Example response

Response

  1. {
  2.     "_shards": {
  3.         "total": 2,
  4.         "successful": 2,
  5.         "failed": 0
  6.     },
  7.     "indices": {
  8.         "remote-index": {
  9.             "shards": {
  10.                 "0": [
  11.                     {
  12.                         "routing": {
  13.                             "state": "STARTED",
  14.                             "primary": true,
  15.                             "node": "q1VxWZnCTICrfRc2bRW3nw"
  16.                         },
  17.                         "segment": {
  18.                             "download": {},
  19.                             "upload": {
  20.                                 "local_refresh_timestamp_in_millis": 1694171634102,
  21.                                 "remote_refresh_timestamp_in_millis": 1694171634102,
  22.                                 "refresh_time_lag_in_millis": 0,
  23.                                 "refresh_lag": 0,
  24.                                 "bytes_lag": 0,
  25.                                 "backpressure_rejection_count": 0,
  26.                                 "consecutive_failure_count": 0,
  27.                                 "total_uploads": {
  28.                                     "started": 5,
  29.                                     "succeeded": 5,
  30.                                     "failed": 0
  31.                                 },
  32.                                 "total_upload_size": {
  33.                                     "started_bytes": 15342,
  34.                                     "succeeded_bytes": 15342,
  35.                                     "failed_bytes": 0
  36.                                 },
  37.                                 "remote_refresh_size_in_bytes": {
  38.                                     "last_successful": 0,
  39.                                     "moving_avg": 3068.4
  40.                                 },
  41.                                 "upload_speed_in_bytes_per_sec": {
  42.                                     "moving_avg": 99988.2
  43.                                 },
  44.                                 "remote_refresh_latency_in_millis": {
  45.                                     "moving_avg": 44.0
  46.                                 }
  47.                             }
  48.                         },
  49.                         "translog": {
  50.                             "upload": {
  51.                                 "last_successful_upload_timestamp": 1694171633644,
  52.                                 "total_uploads": {
  53.                                     "started": 6,
  54.                                     "failed": 0,
  55.                                     "succeeded": 6
  56.                                 },
  57.                                 "total_upload_size": {
  58.                                     "started_bytes": 1932,
  59.                                     "failed_bytes": 0,
  60.                                     "succeeded_bytes": 1932
  61.                                 },
  62.                                 "total_upload_time_in_millis": 21478,
  63.                                 "upload_size_in_bytes": {
  64.                                     "moving_avg": 322.0
  65.                                 },
  66.                                 "upload_speed_in_bytes_per_sec": {
  67.                                     "moving_avg": 2073.8333333333335
  68.                                 },
  69.                                 "upload_time_in_millis": {
  70.                                     "moving_avg": 3579.6666666666665
  71.                                 }
  72.                             },
  73.                             "download": {}
  74.                         }
  75.                     },
  76.                     {
  77.                         "routing": {
  78.                             "state": "STARTED",
  79.                             "primary": false,
  80.                             "node": "EZuen5Y5Sv-eDCLwh9gv-Q"
  81.                         },
  82.                         "segment": {
  83.                             "download": {
  84.                                 "last_sync_timestamp": 1694171634148,
  85.                                 "total_download_size": {
  86.                                     "started_bytes": 15112,
  87.                                     "succeeded_bytes": 15112,
  88.                                     "failed_bytes": 0
  89.                                 },
  90.                                 "download_size_in_bytes": {
  91.                                     "last_successful": 2910,
  92.                                     "moving_avg": 1259.3333333333333
  93.                                 },
  94.                                 "download_speed_in_bytes_per_sec": {
  95.                                     "moving_avg": 382387.3333333333
  96.                                 }
  97.                             },
  98.                             "upload": {}
  99.                         },
  100.                         "translog": {
  101.                             "upload": {},
  102.                             "download": {}
  103.                         }
  104.                     }
  105.                 ]
  106.             }
  107.         }
  108.     }
  109. }

Remote store stats for a local shard

If you want to fetch only shards present on the node serving a Remote Store Stats API request, set the local query parameter to true, as shown in the following example request:

  1. GET _remotestore/stats/<index_name>?local=true

copy