Limit token count token filter

Limit token count token filter

Limits the number of output tokens. The limit filter is commonly used to limit the size of document field values based on token count.

By default, the limit filter keeps only the first token in a stream. For example, the filter can change the token stream [ one, two, three ] to [ one ].

This filter uses Lucene’s LimitTokenCountFilter.

  1.  If you want to limit the size of field values based on
  2. _character length_, use the <<ignore-above,`ignore_above`>> mapping parameter.

Configurable parameters

max_token_count

(Optional, integer) Maximum number of tokens to keep. Once this limit is reached, any remaining tokens are excluded from the output. Defaults to 1.

consume_all_tokens

(Optional, Boolean) If true, the limit filter exhausts the token stream, even if the max_token_count has already been reached. Defaults to false.

Example

The following analyze API request uses the limit filter to keep only the first two tokens in quick fox jumps over lazy dog:

  1. resp = client.indices.analyze(
  2.     tokenizer="standard",
  3.     filter=[
  4.         {
  5.             "type": "limit",
  6.             "max_token_count": 2
  7.         }
  8.     ],
  9.     text="quick fox jumps over lazy dog",
  10. )
  11. print(resp)
  1. response = client.indices.analyze(
  2.   body: {
  3.     tokenizer: 'standard',
  4.     filter: [
  5.       {
  6.         type: 'limit',
  7.         max_token_count: 2
  8.       }
  9.     ],
  10.     text: 'quick fox jumps over lazy dog'
  11.   }
  12. )
  13. puts response
  1. const response = await client.indices.analyze({
  2.   tokenizer: "standard",
  3.   filter: [
  4.     {
  5.       type: "limit",
  6.       max_token_count: 2,
  7.     },
  8.   ],
  9.   text: "quick fox jumps over lazy dog",
  10. });
  11. console.log(response);
  1. GET _analyze
  2. {
  3.   "tokenizer": "standard",
  4.     "filter": [
  5.     {
  6.       "type": "limit",
  7.       "max_token_count": 2
  8.     }
  9.   ],
  10.   "text": "quick fox jumps over lazy dog"
  11. }

The filter produces the following tokens:

  1. [ quick, fox ]

Add to an analyzer

The following create index API request uses the limit filter to configure a new custom analyzer.

  1. resp = client.indices.create(
  2.     index="limit_example",
  3.     settings={
  4.         "analysis": {
  5.             "analyzer": {
  6.                 "standard_one_token_limit": {
  7.                     "tokenizer": "standard",
  8.                     "filter": [
  9.                         "limit"
  10.                     ]
  11.                 }
  12.             }
  13.         }
  14.     },
  15. )
  16. print(resp)
  1. response = client.indices.create(
  2.   index: 'limit_example',
  3.   body: {
  4.     settings: {
  5.       analysis: {
  6.         analyzer: {
  7.           standard_one_token_limit: {
  8.             tokenizer: 'standard',
  9.             filter: [
  10.               'limit'
  11.             ]
  12.           }
  13.         }
  14.       }
  15.     }
  16.   }
  17. )
  18. puts response
  1. const response = await client.indices.create({
  2.   index: "limit_example",
  3.   settings: {
  4.     analysis: {
  5.       analyzer: {
  6.         standard_one_token_limit: {
  7.           tokenizer: "standard",
  8.           filter: ["limit"],
  9.         },
  10.       },
  11.     },
  12.   },
  13. });
  14. console.log(response);
  1. PUT limit_example
  2. {
  3.   "settings": {
  4.     "analysis": {
  5.       "analyzer": {
  6.         "standard_one_token_limit": {
  7.           "tokenizer": "standard",
  8.           "filter": [ "limit" ]
  9.         }
  10.       }
  11.     }
  12.   }
  13. }

Customize

To customize the limit filter, duplicate it to create the basis for a new custom token filter. You can modify the filter using its configurable parameters.

For example, the following request creates a custom limit filter that keeps only the first five tokens of a stream:

  1. resp = client.indices.create(
  2.     index="custom_limit_example",
  3.     settings={
  4.         "analysis": {
  5.             "analyzer": {
  6.                 "whitespace_five_token_limit": {
  7.                     "tokenizer": "whitespace",
  8.                     "filter": [
  9.                         "five_token_limit"
  10.                     ]
  11.                 }
  12.             },
  13.             "filter": {
  14.                 "five_token_limit": {
  15.                     "type": "limit",
  16.                     "max_token_count": 5
  17.                 }
  18.             }
  19.         }
  20.     },
  21. )
  22. print(resp)
  1. response = client.indices.create(
  2.   index: 'custom_limit_example',
  3.   body: {
  4.     settings: {
  5.       analysis: {
  6.         analyzer: {
  7.           whitespace_five_token_limit: {
  8.             tokenizer: 'whitespace',
  9.             filter: [
  10.               'five_token_limit'
  11.             ]
  12.           }
  13.         },
  14.         filter: {
  15.           five_token_limit: {
  16.             type: 'limit',
  17.             max_token_count: 5
  18.           }
  19.         }
  20.       }
  21.     }
  22.   }
  23. )
  24. puts response
  1. const response = await client.indices.create({
  2.   index: "custom_limit_example",
  3.   settings: {
  4.     analysis: {
  5.       analyzer: {
  6.         whitespace_five_token_limit: {
  7.           tokenizer: "whitespace",
  8.           filter: ["five_token_limit"],
  9.         },
  10.       },
  11.       filter: {
  12.         five_token_limit: {
  13.           type: "limit",
  14.           max_token_count: 5,
  15.         },
  16.       },
  17.     },
  18.   },
  19. });
  20. console.log(response);
  1. PUT custom_limit_example
  2. {
  3.   "settings": {
  4.     "analysis": {
  5.       "analyzer": {
  6.         "whitespace_five_token_limit": {
  7.           "tokenizer": "whitespace",
  8.           "filter": [ "five_token_limit" ]
  9.         }
  10.       },
  11.       "filter": {
  12.         "five_token_limit": {
  13.           "type": "limit",
  14.           "max_token_count": 5
  15.         }
  16.       }
  17.     }
  18.   }
  19. }