ignore_malformed

ignore_malformed

Sometimes you don’t have much control over the data that you receive. One user may send a login field that is a date, and another sends a login field that is an email address.

Trying to index the wrong data type into a field throws an exception by default, and rejects the whole document. The ignore_malformed parameter, if set to true, allows the exception to be ignored. The malformed field is not indexed, but other fields in the document are processed normally.

For example:

resp = client.indices.create(
    index="my-index-000001",
    mappings={
        "properties": {
            "number_one": {
                "type": "integer",
                "ignore_malformed": True
            },
            "number_two": {
                "type": "integer"
            }
        }
    },
)
print(resp)
resp1 = client.index(
    index="my-index-000001",
    id="1",
    document={
        "text": "Some text value",
        "number_one": "foo"
    },
)
print(resp1)
resp2 = client.index(
    index="my-index-000001",
    id="2",
    document={
        "text": "Some text value",
        "number_two": "foo"
    },
)
print(resp2)

response = client.indices.create(
  index: 'my-index-000001',
  body: {
    mappings: {
      properties: {
        number_one: {
          type: 'integer',
          ignore_malformed: true
        },
        number_two: {
          type: 'integer'
        }
      }
    }
  }
)
puts response
response = client.index(
  index: 'my-index-000001',
  id: 1,
  body: {
    text: 'Some text value',
    number_one: 'foo'
  }
)
puts response
response = client.index(
  index: 'my-index-000001',
  id: 2,
  body: {
    text: 'Some text value',
    number_two: 'foo'
  }
)
puts response

const response = await client.indices.create({
  index: "my-index-000001",
  mappings: {
    properties: {
      number_one: {
        type: "integer",
        ignore_malformed: true,
      },
      number_two: {
        type: "integer",
      },
    },
  },
});
console.log(response);
const response1 = await client.index({
  index: "my-index-000001",
  id: 1,
  document: {
    text: "Some text value",
    number_one: "foo",
  },
});
console.log(response1);
const response2 = await client.index({
  index: "my-index-000001",
  id: 2,
  document: {
    text: "Some text value",
    number_two: "foo",
  },
});
console.log(response2);

PUT my-index-000001
{
  "mappings": {
    "properties": {
      "number_one": {
        "type": "integer",
        "ignore_malformed": true
      },
      "number_two": {
        "type": "integer"
      }
    }
  }
}
PUT my-index-000001/_doc/1
{
  "text":       "Some text value",
  "number_one": "foo" 
}
PUT my-index-000001/_doc/2
{
  "text":       "Some text value",
  "number_two": "foo" 
}

The ignore_malformed setting is currently supported by the following mapping types:

Numeric

long, integer, short, byte, double, float, half_float, scaled_float

Boolean

boolean

Date

date

Date nanoseconds

date_nanos

Geopoint

geo_point for lat/lon points

Geoshape

geo_shape for complex shapes like polygons

IP

ip for IPv4 and IPv6 addresses

The ignore_malformed setting value can be updated on existing fields using the update mapping API.

Index-level default

The index.mapping.ignore_malformed setting can be set on the index level to ignore malformed content globally across all allowed mapping types. Mapping types that don’t support the setting will ignore it if set on the index level.

resp = client.indices.create(
    index="my-index-000001",
    settings={
        "index.mapping.ignore_malformed": True
    },
    mappings={
        "properties": {
            "number_one": {
                "type": "byte"
            },
            "number_two": {
                "type": "integer",
                "ignore_malformed": False
            }
        }
    },
)
print(resp)

response = client.indices.create(
  index: 'my-index-000001',
  body: {
    settings: {
      'index.mapping.ignore_malformed' => true
    },
    mappings: {
      properties: {
        number_one: {
          type: 'byte'
        },
        number_two: {
          type: 'integer',
          ignore_malformed: false
        }
      }
    }
  }
)
puts response

const response = await client.indices.create({
  index: "my-index-000001",
  settings: {
    "index.mapping.ignore_malformed": true,
  },
  mappings: {
    properties: {
      number_one: {
        type: "byte",
      },
      number_two: {
        type: "integer",
        ignore_malformed: false,
      },
    },
  },
});
console.log(response);

PUT my-index-000001
{
  "settings": {
    "index.mapping.ignore_malformed": true 
  },
  "mappings": {
    "properties": {
      "number_one": { 
        "type": "byte"
      },
      "number_two": {
        "type": "integer",
        "ignore_malformed": false 
      }
    }
  }
}

Dealing with malformed fields

Malformed fields are silently ignored at indexing time when ignore_malformed is turned on. Whenever possible it is recommended to keep the number of documents that have a malformed field contained, or queries on this field will become meaningless. Elasticsearch makes it easy to check how many documents have malformed fields by using exists,term or terms queries on the special _ignored field.

Limits for JSON Objects

You can’t use ignore_malformed with the following data types:

• Nested data type
• Object data type
• Range data types

You also can’t use ignore_malformed to ignore JSON objects submitted to fields of the wrong data type. A JSON object is any data surrounded by curly brackets "{}" and includes data mapped to the nested, object, and range data types.

If you submit a JSON object to an unsupported field, Elasticsearch will return an error and reject the entire document regardless of the ignore_malformed setting.