Collapse search results

Collapse search results

You can use the collapse parameter to collapse search results based on field values. The collapsing is done by selecting only the top sorted document per collapse key.

For example, the following search collapses results by user.id and sorts them by http.response.bytes.

  1. resp = client.search(
  2.     index="my-index-000001",
  3.     query={
  4.         "match": {
  5.             "message": "GET /search"
  6.         }
  7.     },
  8.     collapse={
  9.         "field": "user.id"
  10.     },
  11.     sort=[
  12.         {
  13.             "http.response.bytes": {
  14.                 "order": "desc"
  15.             }
  16.         }
  17.     ],
  18.     from_=0,
  19. )
  20. print(resp)
  1. const response = await client.search({
  2.   index: "my-index-000001",
  3.   query: {
  4.     match: {
  5.       message: "GET /search",
  6.     },
  7.   },
  8.   collapse: {
  9.     field: "user.id",
  10.   },
  11.   sort: [
  12.     {
  13.       "http.response.bytes": {
  14.         order: "desc",
  15.       },
  16.     },
  17.   ],
  18.   from: 0,
  19. });
  20. console.log(response);
  1. GET my-index-000001/_search
  2. {
  3.   "query": {
  4.     "match": {
  5.       "message": "GET /search"
  6.     }
  7.   },
  8.   "collapse": {
  9.     "field": "user.id"         
  10.   },
  11.   "sort": [
  12.     {
  13.       "http.response.bytes": { 
  14.         "order": "desc"
  15.       }
  16.     }
  17.   ],
  18.   "from": 0                    
  19. }

Collapse the result set using the user.id field

Sort the results by http.response.bytes

Define the offset of the first collapsed result

The total number of hits in the response indicates the number of matching documents without collapsing. The total number of distinct group is unknown.

The field used for collapsing must be a single valued keyword or numeric field with doc_values activated.

Collapsing is applied to the top hits only and does not affect aggregations.

Expand collapse results

It is also possible to expand each collapsed top hits with the inner hits option.

  1. resp = client.search(
  2.     index="my-index-000001",
  3.     query={
  4.         "match": {
  5.             "message": "GET /search"
  6.         }
  7.     },
  8.     collapse={
  9.         "field": "user.id",
  10.         "inner_hits": {
  11.             "name": "most_recent",
  12.             "size": 5,
  13.             "sort": [
  14.                 {
  15.                     "@timestamp": "desc"
  16.                 }
  17.             ]
  18.         },
  19.         "max_concurrent_group_searches": 4
  20.     },
  21.     sort=[
  22.         {
  23.             "http.response.bytes": {
  24.                 "order": "desc"
  25.             }
  26.         }
  27.     ],
  28. )
  29. print(resp)
  1. const response = await client.search({
  2.   index: "my-index-000001",
  3.   query: {
  4.     match: {
  5.       message: "GET /search",
  6.     },
  7.   },
  8.   collapse: {
  9.     field: "user.id",
  10.     inner_hits: {
  11.       name: "most_recent",
  12.       size: 5,
  13.       sort: [
  14.         {
  15.           "@timestamp": "desc",
  16.         },
  17.       ],
  18.     },
  19.     max_concurrent_group_searches: 4,
  20.   },
  21.   sort: [
  22.     {
  23.       "http.response.bytes": {
  24.         order: "desc",
  25.       },
  26.     },
  27.   ],
  28. });
  29. console.log(response);
  1. GET /my-index-000001/_search
  2. {
  3.   "query": {
  4.     "match": {
  5.       "message": "GET /search"
  6.     }
  7.   },
  8.   "collapse": {
  9.     "field": "user.id",                       
  10.     "inner_hits": {
  11.       "name": "most_recent",                  
  12.       "size": 5,                              
  13.       "sort": [ { "@timestamp": "desc" } ]    
  14.     },
  15.     "max_concurrent_group_searches": 4        
  16.   },
  17.   "sort": [
  18.     {
  19.       "http.response.bytes": {
  20.         "order": "desc"
  21.       }
  22.     }
  23.   ]
  24. }

Collapse the result set using the user.id field

The name used for the inner hit section in the response

The number of inner_hits to retrieve per collapse key

How to sort the document inside each group

The number of concurrent requests allowed to retrieve the inner_hits per group

See inner hits for the complete list of supported options and the format of the response.

It is also possible to request multiple inner hits for each collapsed hit. This can be useful when you want to get multiple representations of the collapsed hits.

  1. resp = client.search(
  2.     index="my-index-000001",
  3.     query={
  4.         "match": {
  5.             "message": "GET /search"
  6.         }
  7.     },
  8.     collapse={
  9.         "field": "user.id",
  10.         "inner_hits": [
  11.             {
  12.                 "name": "largest_responses",
  13.                 "size": 3,
  14.                 "sort": [
  15.                     {
  16.                         "http.response.bytes": {
  17.                             "order": "desc"
  18.                         }
  19.                     }
  20.                 ]
  21.             },
  22.             {
  23.                 "name": "most_recent",
  24.                 "size": 3,
  25.                 "sort": [
  26.                     {
  27.                         "@timestamp": {
  28.                             "order": "desc"
  29.                         }
  30.                     }
  31.                 ]
  32.             }
  33.         ]
  34.     },
  35.     sort=[
  36.         "http.response.bytes"
  37.     ],
  38. )
  39. print(resp)
  1. const response = await client.search({
  2.   index: "my-index-000001",
  3.   query: {
  4.     match: {
  5.       message: "GET /search",
  6.     },
  7.   },
  8.   collapse: {
  9.     field: "user.id",
  10.     inner_hits: [
  11.       {
  12.         name: "largest_responses",
  13.         size: 3,
  14.         sort: [
  15.           {
  16.             "http.response.bytes": {
  17.               order: "desc",
  18.             },
  19.           },
  20.         ],
  21.       },
  22.       {
  23.         name: "most_recent",
  24.         size: 3,
  25.         sort: [
  26.           {
  27.             "@timestamp": {
  28.               order: "desc",
  29.             },
  30.           },
  31.         ],
  32.       },
  33.     ],
  34.   },
  35.   sort: ["http.response.bytes"],
  36. });
  37. console.log(response);
  1. GET /my-index-000001/_search
  2. {
  3.   "query": {
  4.     "match": {
  5.       "message": "GET /search"
  6.     }
  7.   },
  8.   "collapse": {
  9.     "field": "user.id",                   
  10.     "inner_hits": [
  11.       {
  12.         "name": "largest_responses",      
  13.         "size": 3,
  14.         "sort": [
  15.           {
  16.             "http.response.bytes": {
  17.               "order": "desc"
  18.             }
  19.           }
  20.         ]
  21.       },
  22.       {
  23.         "name": "most_recent",             
  24.         "size": 3,
  25.         "sort": [
  26.           {
  27.             "@timestamp": {
  28.               "order": "desc"
  29.             }
  30.           }
  31.         ]
  32.       }
  33.     ]
  34.   },
  35.   "sort": [
  36.     "http.response.bytes"
  37.   ]
  38. }

Collapse the result set using the user.id field

Return the three largest HTTP responses for the user

Return the three most recent HTTP responses for the user

The expansion of the group is done by sending an additional query for each inner_hit request for each collapsed hit returned in the response. This can significantly slow your search if you have too many groups or inner_hit requests.

The max_concurrent_group_searches request parameter can be used to control the maximum number of concurrent searches allowed in this phase. The default is based on the number of data nodes and the default search thread pool size.

collapse cannot be used in conjunction with scroll.

Collapsing with search_after

Field collapsing can be used with the search_after parameter. Using search_after is only supported when sorting and collapsing on the same field. Secondary sorts are also not allowed. For example, we can collapse and sort on user.id, while paging through the results using search_after:

  1. resp = client.search(
  2.     index="my-index-000001",
  3.     query={
  4.         "match": {
  5.             "message": "GET /search"
  6.         }
  7.     },
  8.     collapse={
  9.         "field": "user.id"
  10.     },
  11.     sort=[
  12.         "user.id"
  13.     ],
  14.     search_after=[
  15.         "dd5ce1ad"
  16.     ],
  17. )
  18. print(resp)
  1. const response = await client.search({
  2.   index: "my-index-000001",
  3.   query: {
  4.     match: {
  5.       message: "GET /search",
  6.     },
  7.   },
  8.   collapse: {
  9.     field: "user.id",
  10.   },
  11.   sort: ["user.id"],
  12.   search_after: ["dd5ce1ad"],
  13. });
  14. console.log(response);
  1. GET /my-index-000001/_search
  2. {
  3.   "query": {
  4.     "match": {
  5.       "message": "GET /search"
  6.     }
  7.   },
  8.   "collapse": {
  9.     "field": "user.id"
  10.   },
  11.   "sort": [ "user.id" ],
  12.   "search_after": ["dd5ce1ad"]
  13. }

Rescore collapse results

You can use field collapsing alongside the rescore search parameter. Rescorers run on every shard for the top-ranked document per collapsed field. To maintain a reliable order, it is recommended to cluster documents sharing the same collapse field value on one shard. This is achieved by assigning the collapse field value as the routing key during indexing:

  1. resp = client.index(
  2.     index="my-index-000001",
  3.     routing="xyz",
  4.     document={
  5.         "@timestamp": "2099-11-15T13:12:00",
  6.         "message": "You know for search!",
  7.         "user.id": "xyz"
  8.     },
  9. )
  10. print(resp)
  1. const response = await client.index({
  2.   index: "my-index-000001",
  3.   routing: "xyz",
  4.   document: {
  5.     "@timestamp": "2099-11-15T13:12:00",
  6.     message: "You know for search!",
  7.     "user.id": "xyz",
  8.   },
  9. });
  10. console.log(response);
  1. POST /my-index-000001/_doc?routing=xyz      
  2. {
  3.   "@timestamp": "2099-11-15T13:12:00",
  4.   "message": "You know for search!",
  5.   "user.id": "xyz"
  6. }

Assign routing with the collapse field value (user.id).

By doing this, you guarantee that only one top document per collapse key gets rescored globally.

The following request utilizes field collapsing on the user.id field and then rescores the top groups with a query rescorer:

  1. resp = client.search(
  2.     index="my-index-000001",
  3.     query={
  4.         "match": {
  5.             "message": "you know for search"
  6.         }
  7.     },
  8.     collapse={
  9.         "field": "user.id"
  10.     },
  11.     rescore={
  12.         "window_size": 50,
  13.         "query": {
  14.             "rescore_query": {
  15.                 "match_phrase": {
  16.                     "message": "you know for search"
  17.                 }
  18.             },
  19.             "query_weight": 0.3,
  20.             "rescore_query_weight": 1.4
  21.         }
  22.     },
  23. )
  24. print(resp)
  1. const response = await client.search({
  2.   index: "my-index-000001",
  3.   query: {
  4.     match: {
  5.       message: "you know for search",
  6.     },
  7.   },
  8.   collapse: {
  9.     field: "user.id",
  10.   },
  11.   rescore: {
  12.     window_size: 50,
  13.     query: {
  14.       rescore_query: {
  15.         match_phrase: {
  16.           message: "you know for search",
  17.         },
  18.       },
  19.       query_weight: 0.3,
  20.       rescore_query_weight: 1.4,
  21.     },
  22.   },
  23. });
  24. console.log(response);
  1. GET /my-index-000001/_search
  2. {
  3.   "query": {
  4.     "match": {
  5.       "message": "you know for search"
  6.     }
  7.   },
  8.   "collapse": {
  9.     "field": "user.id"
  10.   },
  11.   "rescore" : {
  12.       "window_size" : 50,
  13.       "query" : {
  14.          "rescore_query" : {
  15.             "match_phrase": {
  16.                 "message": "you know for search"
  17.             }
  18.          },
  19.          "query_weight" : 0.3,
  20.          "rescore_query_weight" : 1.4
  21.       }
  22.    }
  23. }

Rescorers are not applied to inner hits.

Second level of collapsing

A second level of collapsing is also supported and is applied to inner_hits.

For example, the following search collapses results by geo.country_name. Within each geo.country_name, inner hits are collapsed by user.id.

Second level of collapsing doesn’t allow inner_hits.

  1. resp = client.search(
  2.     index="my-index-000001",
  3.     query={
  4.         "match": {
  5.             "message": "GET /search"
  6.         }
  7.     },
  8.     collapse={
  9.         "field": "geo.country_name",
  10.         "inner_hits": {
  11.             "name": "by_location",
  12.             "collapse": {
  13.                 "field": "user.id"
  14.             },
  15.             "size": 3
  16.         }
  17.     },
  18. )
  19. print(resp)
  1. const response = await client.search({
  2.   index: "my-index-000001",
  3.   query: {
  4.     match: {
  5.       message: "GET /search",
  6.     },
  7.   },
  8.   collapse: {
  9.     field: "geo.country_name",
  10.     inner_hits: {
  11.       name: "by_location",
  12.       collapse: {
  13.         field: "user.id",
  14.       },
  15.       size: 3,
  16.     },
  17.   },
  18. });
  19. console.log(response);
  1. GET /my-index-000001/_search
  2. {
  3.   "query": {
  4.     "match": {
  5.       "message": "GET /search"
  6.     }
  7.   },
  8.   "collapse": {
  9.     "field": "geo.country_name",
  10.     "inner_hits": {
  11.       "name": "by_location",
  12.       "collapse": { "field": "user.id" },
  13.       "size": 3
  14.     }
  15.   }
  16. }
  1. {
  2.   "hits" : {
  3.     "hits" : [
  4.       {
  5.         "_index" : "my-index-000001",
  6.         "_id" : "oX9uXXoB0da05OCR3adK",
  7.         "_score" : 0.5753642,
  8.         "_source" : {
  9.           "@timestamp" : "2099-11-15T14:12:12",
  10.           "geo" : {
  11.             "country_name" : "Amsterdam"
  12.           },
  13.           "http" : {
  14.             "request" : {
  15.               "method" : "get"
  16.             },
  17.             "response" : {
  18.               "bytes" : 1070000,
  19.               "status_code" : 200
  20.             },
  21.             "version" : "1.1"
  22.           },
  23.           "message" : "GET /search HTTP/1.1 200 1070000",
  24.           "source" : {
  25.             "ip" : "127.0.0.1"
  26.           },
  27.           "user" : {
  28.             "id" : "kimchy"
  29.           }
  30.         },
  31.         "fields" : {
  32.           "geo.country_name" : [
  33.             "Amsterdam"
  34.           ]
  35.         },
  36.         "inner_hits" : {
  37.           "by_location" : {
  38.             "hits" : {
  39.               "total" : {
  40.                 "value" : 1,
  41.                 "relation" : "eq"
  42.               },
  43.               "max_score" : 0.5753642,
  44.               "hits" : [
  45.                 {
  46.                   "_index" : "my-index-000001",
  47.                   "_id" : "oX9uXXoB0da05OCR3adK",
  48.                   "_score" : 0.5753642,
  49.                   "_source" : {
  50.                     "@timestamp" : "2099-11-15T14:12:12",
  51.                     "geo" : {
  52.                       "country_name" : "Amsterdam"
  53.                     },
  54.                     "http" : {
  55.                       "request" : {
  56.                         "method" : "get"
  57.                       },
  58.                       "response" : {
  59.                         "bytes" : 1070000,
  60.                         "status_code" : 200
  61.                       },
  62.                       "version" : "1.1"
  63.                     },
  64.                     "message" : "GET /search HTTP/1.1 200 1070000",
  65.                     "source" : {
  66.                       "ip" : "127.0.0.1"
  67.                     },
  68.                     "user" : {
  69.                       "id" : "kimchy"
  70.                     }
  71.                   },
  72.                   "fields" : {
  73.                     "user.id" : [
  74.                       "kimchy"
  75.                     ]
  76.                   }
  77.                 }
  78.               ]
  79.             }
  80.           }
  81.         }
  82.       }
  83.     ]
  84.   }
  85. }

Track Scores

When collapse is used with sort on a field, scores are not computed. Setting track_scores to true instructs Elasticsearch to compute and track scores.