Truncate token filter

Truncate token filter

Truncates tokens that exceed a specified character limit. This limit defaults to 10 but can be customized using the length parameter.

For example, you can use the truncate filter to shorten all tokens to 3 characters or fewer, changing jumping fox to jum fox.

This filter uses Lucene’s TruncateTokenFilter.

Example

The following analyze API request uses the truncate filter to shorten tokens that exceed 10 characters in the quinquennial extravaganza carried on:

  1. resp = client.indices.analyze(
  2.     tokenizer="whitespace",
  3.     filter=[
  4.         "truncate"
  5.     ],
  6.     text="the quinquennial extravaganza carried on",
  7. )
  8. print(resp)
  1. response = client.indices.analyze(
  2.   body: {
  3.     tokenizer: 'whitespace',
  4.     filter: [
  5.       'truncate'
  6.     ],
  7.     text: 'the quinquennial extravaganza carried on'
  8.   }
  9. )
  10. puts response
  1. const response = await client.indices.analyze({
  2.   tokenizer: "whitespace",
  3.   filter: ["truncate"],
  4.   text: "the quinquennial extravaganza carried on",
  5. });
  6. console.log(response);
  1. GET _analyze
  2. {
  3.   "tokenizer" : "whitespace",
  4.   "filter" : ["truncate"],
  5.   "text" : "the quinquennial extravaganza carried on"
  6. }

The filter produces the following tokens:

  1. [ the, quinquenni, extravagan, carried, on ]

Add to an analyzer

The following create index API request uses the truncate filter to configure a new custom analyzer.

  1. resp = client.indices.create(
  2.     index="custom_truncate_example",
  3.     settings={
  4.         "analysis": {
  5.             "analyzer": {
  6.                 "standard_truncate": {
  7.                     "tokenizer": "standard",
  8.                     "filter": [
  9.                         "truncate"
  10.                     ]
  11.                 }
  12.             }
  13.         }
  14.     },
  15. )
  16. print(resp)
  1. response = client.indices.create(
  2.   index: 'custom_truncate_example',
  3.   body: {
  4.     settings: {
  5.       analysis: {
  6.         analyzer: {
  7.           standard_truncate: {
  8.             tokenizer: 'standard',
  9.             filter: [
  10.               'truncate'
  11.             ]
  12.           }
  13.         }
  14.       }
  15.     }
  16.   }
  17. )
  18. puts response
  1. const response = await client.indices.create({
  2.   index: "custom_truncate_example",
  3.   settings: {
  4.     analysis: {
  5.       analyzer: {
  6.         standard_truncate: {
  7.           tokenizer: "standard",
  8.           filter: ["truncate"],
  9.         },
  10.       },
  11.     },
  12.   },
  13. });
  14. console.log(response);
  1. PUT custom_truncate_example
  2. {
  3.   "settings" : {
  4.     "analysis" : {
  5.       "analyzer" : {
  6.         "standard_truncate" : {
  7.         "tokenizer" : "standard",
  8.         "filter" : ["truncate"]
  9.         }
  10.       }
  11.     }
  12.   }
  13. }

Configurable parameters

length

(Optional, integer) Character limit for each token. Tokens exceeding this limit are truncated. Defaults to 10.

Customize

To customize the truncate filter, duplicate it to create the basis for a new custom token filter. You can modify the filter using its configurable parameters.

For example, the following request creates a custom truncate filter, 5_char_trunc, that shortens tokens to a length of 5 or fewer characters:

  1. resp = client.indices.create(
  2.     index="5_char_words_example",
  3.     settings={
  4.         "analysis": {
  5.             "analyzer": {
  6.                 "lowercase_5_char": {
  7.                     "tokenizer": "lowercase",
  8.                     "filter": [
  9.                         "5_char_trunc"
  10.                     ]
  11.                 }
  12.             },
  13.             "filter": {
  14.                 "5_char_trunc": {
  15.                     "type": "truncate",
  16.                     "length": 5
  17.                 }
  18.             }
  19.         }
  20.     },
  21. )
  22. print(resp)
  1. response = client.indices.create(
  2.   index: '5_char_words_example',
  3.   body: {
  4.     settings: {
  5.       analysis: {
  6.         analyzer: {
  7.           "lowercase_5_char": {
  8.             tokenizer: 'lowercase',
  9.             filter: [
  10.               '5_char_trunc'
  11.             ]
  12.           }
  13.         },
  14.         filter: {
  15.           "5_char_trunc": {
  16.             type: 'truncate',
  17.             length: 5
  18.           }
  19.         }
  20.       }
  21.     }
  22.   }
  23. )
  24. puts response
  1. const response = await client.indices.create({
  2.   index: "5_char_words_example",
  3.   settings: {
  4.     analysis: {
  5.       analyzer: {
  6.         lowercase_5_char: {
  7.           tokenizer: "lowercase",
  8.           filter: ["5_char_trunc"],
  9.         },
  10.       },
  11.       filter: {
  12.         "5_char_trunc": {
  13.           type: "truncate",
  14.           length: 5,
  15.         },
  16.       },
  17.     },
  18.   },
  19. });
  20. console.log(response);
  1. PUT 5_char_words_example
  2. {
  3.   "settings": {
  4.     "analysis": {
  5.       "analyzer": {
  6.         "lowercase_5_char": {
  7.           "tokenizer": "lowercase",
  8.           "filter": [ "5_char_trunc" ]
  9.         }
  10.       },
  11.       "filter": {
  12.         "5_char_trunc": {
  13.           "type": "truncate",
  14.           "length": 5
  15.         }
  16.       }
  17.     }
  18.   }
  19. }