Stemmer override token filter

Stemmer override token filter

Overrides stemming algorithms, by applying a custom mapping, then protecting these terms from being modified by stemmers. Must be placed before any stemming filters.

Rules are mappings in the form of token1[, ..., tokenN] => override.

SettingDescription

rules

A list of mapping rules to use.

rules_path

A path (either relative to config location, or absolute) to a list of mappings.

Here is an example:

  1. resp = client.indices.create(
  2.     index="my-index-000001",
  3.     settings={
  4.         "analysis": {
  5.             "analyzer": {
  6.                 "my_analyzer": {
  7.                     "tokenizer": "standard",
  8.                     "filter": [
  9.                         "lowercase",
  10.                         "custom_stems",
  11.                         "porter_stem"
  12.                     ]
  13.                 }
  14.             },
  15.             "filter": {
  16.                 "custom_stems": {
  17.                     "type": "stemmer_override",
  18.                     "rules_path": "analysis/stemmer_override.txt"
  19.                 }
  20.             }
  21.         }
  22.     },
  23. )
  24. print(resp)
  1. response = client.indices.create(
  2.   index: 'my-index-000001',
  3.   body: {
  4.     settings: {
  5.       analysis: {
  6.         analyzer: {
  7.           my_analyzer: {
  8.             tokenizer: 'standard',
  9.             filter: [
  10.               'lowercase',
  11.               'custom_stems',
  12.               'porter_stem'
  13.             ]
  14.           }
  15.         },
  16.         filter: {
  17.           custom_stems: {
  18.             type: 'stemmer_override',
  19.             rules_path: 'analysis/stemmer_override.txt'
  20.           }
  21.         }
  22.       }
  23.     }
  24.   }
  25. )
  26. puts response
  1. const response = await client.indices.create({
  2.   index: "my-index-000001",
  3.   settings: {
  4.     analysis: {
  5.       analyzer: {
  6.         my_analyzer: {
  7.           tokenizer: "standard",
  8.           filter: ["lowercase", "custom_stems", "porter_stem"],
  9.         },
  10.       },
  11.       filter: {
  12.         custom_stems: {
  13.           type: "stemmer_override",
  14.           rules_path: "analysis/stemmer_override.txt",
  15.         },
  16.       },
  17.     },
  18.   },
  19. });
  20. console.log(response);
  1. PUT /my-index-000001
  2. {
  3.   "settings": {
  4.     "analysis": {
  5.       "analyzer": {
  6.         "my_analyzer": {
  7.           "tokenizer": "standard",
  8.           "filter": [ "lowercase", "custom_stems", "porter_stem" ]
  9.         }
  10.       },
  11.       "filter": {
  12.         "custom_stems": {
  13.           "type": "stemmer_override",
  14.           "rules_path": "analysis/stemmer_override.txt"
  15.         }
  16.       }
  17.     }
  18.   }
  19. }

Where the file looks like:

  1. running, runs => run
  3. stemmer => stemmer

You can also define the overrides rules inline:

  1. resp = client.indices.create(
  2.     index="my-index-000001",
  3.     settings={
  4.         "analysis": {
  5.             "analyzer": {
  6.                 "my_analyzer": {
  7.                     "tokenizer": "standard",
  8.                     "filter": [
  9.                         "lowercase",
  10.                         "custom_stems",
  11.                         "porter_stem"
  12.                     ]
  13.                 }
  14.             },
  15.             "filter": {
  16.                 "custom_stems": {
  17.                     "type": "stemmer_override",
  18.                     "rules": [
  19.                         "running, runs => run",
  20.                         "stemmer => stemmer"
  21.                     ]
  22.                 }
  23.             }
  24.         }
  25.     },
  26. )
  27. print(resp)
  1. response = client.indices.create(
  2.   index: 'my-index-000001',
  3.   body: {
  4.     settings: {
  5.       analysis: {
  6.         analyzer: {
  7.           my_analyzer: {
  8.             tokenizer: 'standard',
  9.             filter: [
  10.               'lowercase',
  11.               'custom_stems',
  12.               'porter_stem'
  13.             ]
  14.           }
  15.         },
  16.         filter: {
  17.           custom_stems: {
  18.             type: 'stemmer_override',
  19.             rules: [
  20.               'running, runs => run',
  21.               'stemmer => stemmer'
  22.             ]
  23.           }
  24.         }
  25.       }
  26.     }
  27.   }
  28. )
  29. puts response
  1. const response = await client.indices.create({
  2.   index: "my-index-000001",
  3.   settings: {
  4.     analysis: {
  5.       analyzer: {
  6.         my_analyzer: {
  7.           tokenizer: "standard",
  8.           filter: ["lowercase", "custom_stems", "porter_stem"],
  9.         },
  10.       },
  11.       filter: {
  12.         custom_stems: {
  13.           type: "stemmer_override",
  14.           rules: ["running, runs => run", "stemmer => stemmer"],
  15.         },
  16.       },
  17.     },
  18.   },
  19. });
  20. console.log(response);
  1. PUT /my-index-000001
  2. {
  3.   "settings": {
  4.     "analysis": {
  5.       "analyzer": {
  6.         "my_analyzer": {
  7.           "tokenizer": "standard",
  8.           "filter": [ "lowercase", "custom_stems", "porter_stem" ]
  9.         }
  10.       },
  11.       "filter": {
  12.         "custom_stems": {
  13.           "type": "stemmer_override",
  14.           "rules": [
  15.             "running, runs => run",
  16.             "stemmer => stemmer"
  17.           ]
  18.         }
  19.       }
  20.     }
  21.   }
  22. }