analyzer

analyzer

Only text fields support the analyzer mapping parameter.

The analyzer parameter specifies the analyzer used for text analysis when indexing or searching a text field.

Unless overridden with the search_analyzer mapping parameter, this analyzer is used for both index and search analysis. See Specify an analyzer.

We recommend testing analyzers before using them in production. See Test an analyzer.

The analyzer setting can not be updated on existing fields using the update mapping API.

search_quote_analyzer

The search_quote_analyzer setting allows you to specify an analyzer for phrases, this is particularly useful when dealing with disabling stop words for phrase queries.

To disable stop words for phrases a field utilising three analyzer settings will be required:

  1. An analyzer setting for indexing all terms including stop words
  2. A search_analyzer setting for non-phrase queries that will remove stop words
  3. A search_quote_analyzer setting for phrase queries that will not remove stop words
  1. resp = client.indices.create(
  2.     index="my-index-000001",
  3.     settings={
  4.         "analysis": {
  5.             "analyzer": {
  6.                 "my_analyzer": {
  7.                     "type": "custom",
  8.                     "tokenizer": "standard",
  9.                     "filter": [
  10.                         "lowercase"
  11.                     ]
  12.                 },
  13.                 "my_stop_analyzer": {
  14.                     "type": "custom",
  15.                     "tokenizer": "standard",
  16.                     "filter": [
  17.                         "lowercase",
  18.                         "english_stop"
  19.                     ]
  20.                 }
  21.             },
  22.             "filter": {
  23.                 "english_stop": {
  24.                     "type": "stop",
  25.                     "stopwords": "_english_"
  26.                 }
  27.             }
  28.         }
  29.     },
  30.     mappings={
  31.         "properties": {
  32.             "title": {
  33.                 "type": "text",
  34.                 "analyzer": "my_analyzer",
  35.                 "search_analyzer": "my_stop_analyzer",
  36.                 "search_quote_analyzer": "my_analyzer"
  37.             }
  38.         }
  39.     },
  40. )
  41. print(resp)
  43. resp1 = client.index(
  44.     index="my-index-000001",
  45.     id="1",
  46.     document={
  47.         "title": "The Quick Brown Fox"
  48.     },
  49. )
  50. print(resp1)
  52. resp2 = client.index(
  53.     index="my-index-000001",
  54.     id="2",
  55.     document={
  56.         "title": "A Quick Brown Fox"
  57.     },
  58. )
  59. print(resp2)
  61. resp3 = client.search(
  62.     index="my-index-000001",
  63.     query={
  64.         "query_string": {
  65.             "query": "\"the quick brown fox\""
  66.         }
  67.     },
  68. )
  69. print(resp3)
  1. response = client.indices.create(
  2.   index: 'my-index-000001',
  3.   body: {
  4.     settings: {
  5.       analysis: {
  6.         analyzer: {
  7.           my_analyzer: {
  8.             type: 'custom',
  9.             tokenizer: 'standard',
  10.             filter: [
  11.               'lowercase'
  12.             ]
  13.           },
  14.           my_stop_analyzer: {
  15.             type: 'custom',
  16.             tokenizer: 'standard',
  17.             filter: [
  18.               'lowercase',
  19.               'english_stop'
  20.             ]
  21.           }
  22.         },
  23.         filter: {
  24.           english_stop: {
  25.             type: 'stop',
  26.             stopwords: '_english_'
  27.           }
  28.         }
  29.       }
  30.     },
  31.     mappings: {
  32.       properties: {
  33.         title: {
  34.           type: 'text',
  35.           analyzer: 'my_analyzer',
  36.           search_analyzer: 'my_stop_analyzer',
  37.           search_quote_analyzer: 'my_analyzer'
  38.         }
  39.       }
  40.     }
  41.   }
  42. )
  43. puts response
  45. response = client.index(
  46.   index: 'my-index-000001',
  47.   id: 1,
  48.   body: {
  49.     title: 'The Quick Brown Fox'
  50.   }
  51. )
  52. puts response
  54. response = client.index(
  55.   index: 'my-index-000001',
  56.   id: 2,
  57.   body: {
  58.     title: 'A Quick Brown Fox'
  59.   }
  60. )
  61. puts response
  63. response = client.search(
  64.   index: 'my-index-000001',
  65.   body: {
  66.     query: {
  67.       query_string: {
  68.         query: '"the quick brown fox"'
  69.       }
  70.     }
  71.   }
  72. )
  73. puts response
  1. const response = await client.indices.create({
  2.   index: "my-index-000001",
  3.   settings: {
  4.     analysis: {
  5.       analyzer: {
  6.         my_analyzer: {
  7.           type: "custom",
  8.           tokenizer: "standard",
  9.           filter: ["lowercase"],
  10.         },
  11.         my_stop_analyzer: {
  12.           type: "custom",
  13.           tokenizer: "standard",
  14.           filter: ["lowercase", "english_stop"],
  15.         },
  16.       },
  17.       filter: {
  18.         english_stop: {
  19.           type: "stop",
  20.           stopwords: "_english_",
  21.         },
  22.       },
  23.     },
  24.   },
  25.   mappings: {
  26.     properties: {
  27.       title: {
  28.         type: "text",
  29.         analyzer: "my_analyzer",
  30.         search_analyzer: "my_stop_analyzer",
  31.         search_quote_analyzer: "my_analyzer",
  32.       },
  33.     },
  34.   },
  35. });
  36. console.log(response);
  38. const response1 = await client.index({
  39.   index: "my-index-000001",
  40.   id: 1,
  41.   document: {
  42.     title: "The Quick Brown Fox",
  43.   },
  44. });
  45. console.log(response1);
  47. const response2 = await client.index({
  48.   index: "my-index-000001",
  49.   id: 2,
  50.   document: {
  51.     title: "A Quick Brown Fox",
  52.   },
  53. });
  54. console.log(response2);
  56. const response3 = await client.search({
  57.   index: "my-index-000001",
  58.   query: {
  59.     query_string: {
  60.       query: '"the quick brown fox"',
  61.     },
  62.   },
  63. });
  64. console.log(response3);
  1. PUT my-index-000001
  2. {
  3.    "settings":{
  4.       "analysis":{
  5.          "analyzer":{
  6.             "my_analyzer":{ 
  7.                "type":"custom",
  8.                "tokenizer":"standard",
  9.                "filter":[
  10.                   "lowercase"
  11.                ]
  12.             },
  13.             "my_stop_analyzer":{ 
  14.                "type":"custom",
  15.                "tokenizer":"standard",
  16.                "filter":[
  17.                   "lowercase",
  18.                   "english_stop"
  19.                ]
  20.             }
  21.          },
  22.          "filter":{
  23.             "english_stop":{
  24.                "type":"stop",
  25.                "stopwords":"_english_"
  26.             }
  27.          }
  28.       }
  29.    },
  30.    "mappings":{
  31.        "properties":{
  32.           "title": {
  33.              "type":"text",
  34.              "analyzer":"my_analyzer", 
  35.              "search_analyzer":"my_stop_analyzer", 
  36.              "search_quote_analyzer":"my_analyzer" 
  37.          }
  38.       }
  39.    }
  40. }
  42. PUT my-index-000001/_doc/1
  43. {
  44.    "title":"The Quick Brown Fox"
  45. }
  47. PUT my-index-000001/_doc/2
  48. {
  49.    "title":"A Quick Brown Fox"
  50. }
  52. GET my-index-000001/_search
  53. {
  54.    "query":{
  55.       "query_string":{
  56.          "query":"\"the quick brown fox\"" 
  57.       }
  58.    }
  59. }

The search_quote_analyzer setting can be updated on existing fields using the update mapping API.

my_analyzer analyzer which tokens all terms including stop words

my_stop_analyzer analyzer which removes stop words

analyzer setting that points to the my_analyzer analyzer which will be used at index time

search_analyzer setting that points to the my_stop_analyzer and removes stop words for non-phrase queries

search_quote_analyzer setting that points to the my_analyzer analyzer and ensures that stop words are not removed from phrase queries

Since the query is wrapped in quotes it is detected as a phrase query therefore the search_quote_analyzer kicks in and ensures the stop words are not removed from the query. The my_analyzer analyzer will then return the following tokens [the, quick, brown, fox] which will match one of the documents. Meanwhile term queries will be analyzed with the my_stop_analyzer analyzer which will filter out stop words. So a search for either The quick brown fox or A quick brown fox will return both documents since both documents contain the following tokens [quick, brown, fox]. Without the search_quote_analyzer it would not be possible to do exact matches for phrase queries as the stop words from phrase queries would be removed resulting in both documents matching.