Search: add format support for date range filter and queries

When the date format is defined in mapping, you can not use another format when querying using range date query or filter.

For example, this won't work:

```
DELETE /test

PUT /test/t/1
{
  "date": "2014-01-01"
}

GET /test/_search
{
  "query": {
    "filtered": {
      "filter": {
        "range": {
          "date": {
            "from": "01/01/2014"
          }
        }
      }
    }
  }
}
```

It causes:

```
Caused by: org.elasticsearch.ElasticsearchParseException: failed to parse date field [01/01/2014], tried both date format [dateOptionalTime], and timestamp number
```

It could be nice if we can support at query time another date format just like we support `analyzer` at search time on String fields.

Something like:

```
GET /test/_search
{
  "query": {
    "filtered": {
      "filter": {
        "range": {
          "date": {
            "from": "01/01/2014",
            "format": "dd/MM/yyyy"
          }
        }
      }
    }
  }
}
```

Same for queries:

```
GET /test/_search
{
  "query": {
    "range": {
      "date": {
        "from": "01/01/2014",
        "format": "dd/MM/yyyy"
      }
    }
  }
}
```

Closes #7189.

Author: dadoonet

docs/reference/query-dsl/filters/range-filter.asciidoc

@@ -30,6 +30,9 @@ The `range` filter accepts the following parameters:
 `lte`::     Less-than or equal to
 `lt`::      Less-than
 
+[float]
+==== Date options
+
 When applied on `date` fields the `range` filter accepts also a `time_zone` parameter.
 The `time_zone` parameter will be applied to your input lower and upper bounds and will
 move them to UTC time based date:
@@ -56,6 +59,28 @@ In the above example, `gte` will be actually moved to `2011-12-31T23:00:00` UTC
 NOTE: if you give a date with a timezone explicitly defined and use the `time_zone` parameter, `time_zone` will be
 ignored. For example, setting `from` to `2012-01-01T00:00:00+01:00` with `"time_zone":"+10:00"` will still use `+01:00` time zone.
 
+coming[1.5.0,New feature added]
+
+When applied on `date` fields the `range` filter accepts also a `format` parameter.
+The `format` parameter will help support another date format than the one defined in mapping:
+
+[source,js]
+--------------------------------------------------
+{
+    "constant_score": {
+        "filter": {
+            "range" : {
+                "born" : {
+                    "gte": "01/01/2012",
+                    "lte": "2013",
+                    "format": "dd/MM/yyyy||yyyy"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+
 [float]
 ==== Execution
 

docs/reference/query-dsl/queries/range-query.asciidoc

@@ -29,6 +29,9 @@ The `range` query accepts the following parameters:
 `lt`::  	Less-than
 `boost`:: 	Sets the boost value of the query, defaults to `1.0`
 
+[float]
+==== Date options
+
 When applied on `date` fields the `range` filter accepts also a `time_zone` parameter.
 The `time_zone` parameter will be applied to your input lower and upper bounds and will
 move them to UTC time based date:
@@ -51,3 +54,20 @@ In the above example, `gte` will be actually moved to `2011-12-31T23:00:00` UTC
 NOTE: if you give a date with a timezone explicitly defined and use the `time_zone` parameter, `time_zone` will be
 ignored. For example, setting `from` to `2012-01-01T00:00:00+01:00` with `"time_zone":"+10:00"` will still use `+01:00` time zone.
 
+coming[1.5.0,New feature added]
+
+When applied on `date` fields the `range` query accepts also a `format` parameter.
+The `format` parameter will help support another date format than the one defined in mapping:
+
+[source,js]
+--------------------------------------------------
+{
+    "range" : {
+        "born" : {
+            "gte": "01/01/2012",
+            "lte": "2013",
+            "format": "dd/MM/yyyy||yyyy"
+        }
+    }
+}
+--------------------------------------------------

src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java

@@ -300,19 +300,23 @@ public long parseToMilliseconds(Object value, @Nullable QueryParseContext contex
     }
 
     public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper) {
-        return parseToMilliseconds(value, context, includeUpper, null);
+        return parseToMilliseconds(value, context, includeUpper, null, dateMathParser);
     }
 
-    public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone) {
+    public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
         if (value instanceof Number) {
             return ((Number) value).longValue();
         }
-        return parseToMilliseconds(convertToString(value), context, includeUpper, zone);
+        return parseToMilliseconds(convertToString(value), context, includeUpper, zone, forcedDateParser);
     }
 
-    public long parseToMilliseconds(String value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone) {
+    public long parseToMilliseconds(String value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
         long now = context == null ? System.currentTimeMillis() : context.nowInMillis();
-        long time = includeUpper && roundCeil ? dateMathParser.parseRoundCeil(value, now, zone) : dateMathParser.parse(value, now, zone);
+        DateMathParser dateParser = dateMathParser;
+        if (forcedDateParser != null) {
+            dateParser = forcedDateParser;
+        }
+        long time = includeUpper && roundCeil ? dateParser.parseRoundCeil(value, now, zone) : dateParser.parse(value, now, zone);
         return time;
     }
 
@@ -325,28 +329,28 @@ public Filter termFilter(Object value, @Nullable QueryParseContext context) {
 
     @Override
     public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, context);
+        return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, null, context);
     }
 
-    public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context) {
+    public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context) {
         return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
-                lowerTerm == null ? null : parseToMilliseconds(lowerTerm, context, false, timeZone),
-                upperTerm == null ? null : parseToMilliseconds(upperTerm, context, includeUpper, timeZone),
+                lowerTerm == null ? null : parseToMilliseconds(lowerTerm, context, false, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
+                upperTerm == null ? null : parseToMilliseconds(upperTerm, context, includeUpper, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
                 includeLower, includeUpper);
     }
 
     @Override
     public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return rangeFilter(lowerTerm, upperTerm, includeLower, includeUpper, null, context, null);
+        return rangeFilter(lowerTerm, upperTerm, includeLower, includeUpper, null, null, context, null);
     }
 
-    public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
-        return rangeFilter(null, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, context, explicitCaching);
+    public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
+        return rangeFilter(null, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, context, explicitCaching);
     }
 
     @Override
     public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return rangeFilter(parseContext, lowerTerm, upperTerm, includeLower, includeUpper, null, context, null);
+        return rangeFilter(parseContext, lowerTerm, upperTerm, includeLower, includeUpper, null, null, context, null);
     }
 
     /*
@@ -355,7 +359,7 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
      * - the object to parse is a String (does not apply to ms since epoch which are UTC based time values)
      * - the String to parse does not have already a timezone defined (ie. `2014-01-01T00:00:00+03:00`)
      */
-    public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
+    public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
         boolean cache;
         boolean cacheable = true;
         Long lowerVal = null;
@@ -366,7 +370,7 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
             } else {
                 String value = convertToString(lowerTerm);
                 cacheable = !hasDateExpressionWithNoRounding(value);
-                lowerVal = parseToMilliseconds(value, context, false, timeZone);
+                lowerVal = parseToMilliseconds(value, context, false, timeZone, forcedDateParser);
             }
         }
         if (upperTerm != null) {
@@ -375,7 +379,7 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
             } else {
                 String value = convertToString(upperTerm);
                 cacheable = cacheable && !hasDateExpressionWithNoRounding(value);
-                upperVal = parseToMilliseconds(value, context, includeUpper, timeZone);
+                upperVal = parseToMilliseconds(value, context, includeUpper, timeZone, forcedDateParser);
             }
         }
 

src/main/java/org/elasticsearch/index/query/RangeFilterParser.java

@@ -23,6 +23,7 @@
 import org.apache.lucene.search.TermRangeFilter;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.joda.DateMathParser;
+import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
@@ -64,6 +65,7 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
         boolean includeLower = true;
         boolean includeUpper = true;
         DateTimeZone timeZone = null;
+        DateMathParser forcedDateParser = null;
         String execution = "index";
 
         String filterName = null;
@@ -100,6 +102,8 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
                             includeUpper = true;
                         } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) {
                             timeZone = DateMathParser.parseZone(parser.text());
+                        } else if ("format".equals(currentFieldName)) {
+                            forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT);
                         } else {
                             throw new QueryParsingException(parseContext.index(), "[range] filter does not support [" + currentFieldName + "]");
                         }
@@ -138,7 +142,7 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
                         if ((from instanceof Number || to instanceof Number) && timeZone != null) {
                             throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]");
                         }
-                        filter = ((DateFieldMapper) mapper).rangeFilter(from, to, includeLower, includeUpper, timeZone, parseContext, explicitlyCached);
+                        filter = ((DateFieldMapper) mapper).rangeFilter(from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext, explicitlyCached);
                     } else  {
                         if (timeZone != null) {
                             throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]");
@@ -157,7 +161,7 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
                         if ((from instanceof Number || to instanceof Number) && timeZone != null) {
                             throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]");
                         }
-                        filter = ((DateFieldMapper) mapper).rangeFilter(parseContext, from, to, includeLower, includeUpper, timeZone, parseContext, explicitlyCached);
+                        filter = ((DateFieldMapper) mapper).rangeFilter(parseContext, from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext, explicitlyCached);
                     } else {
                         if (timeZone != null) {
                             throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]");

src/main/java/org/elasticsearch/index/query/RangeQueryParser.java

@@ -23,6 +23,7 @@
 import org.apache.lucene.search.TermRangeQuery;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.joda.DateMathParser;
+import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -69,6 +70,7 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
         boolean includeLower = true;
         boolean includeUpper = true;
         DateTimeZone timeZone = null;
+        DateMathParser forcedDateParser = null;
         float boost = 1.0f;
         String queryName = null;
 
@@ -103,6 +105,8 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
                     timeZone = DateMathParser.parseZone(parser.text());
                 } else if ("_name".equals(currentFieldName)) {
                     queryName = parser.text();
+                } else if ("format".equals(currentFieldName)) {
+                    forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT);
                 } else {
                     throw new QueryParsingException(parseContext.index(), "[range] query does not support [" + currentFieldName + "]");
                 }
@@ -124,7 +128,7 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
                     if ((from instanceof Number || to instanceof Number) && timeZone != null) {
                         throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]");
                     }
-                    query = ((DateFieldMapper) mapper).rangeQuery(from, to, includeLower, includeUpper, timeZone, parseContext);
+                    query = ((DateFieldMapper) mapper).rangeQuery(from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext);
                 } else  {
                     if (timeZone != null) {
                         throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]");

src/test/java/org/elasticsearch/index/query/IndexQueryParserFilterDateRangeFormatTests.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.inject.Injector;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.service.IndexService;
+import org.elasticsearch.test.ElasticsearchSingleNodeTest;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
+import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
+import static org.hamcrest.Matchers.*;
+
+/**
+ *
+ */
+public class IndexQueryParserFilterDateRangeFormatTests extends ElasticsearchSingleNodeTest {
+
+    private Injector injector;
+    private IndexQueryParserService queryParser;
+
+    @Before
+    public void setup() throws IOException {
+        IndexService indexService = createIndex("test");
+        injector = indexService.injector();
+
+        MapperService mapperService = indexService.mapperService();
+        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/query/mapping.json");
+        mapperService.merge("person", new CompressedString(mapping), true);
+        mapperService.documentMapper("person").parse(new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/query/data.json")));
+        queryParser = injector.getInstance(IndexQueryParserService.class);
+    }
+
+    private IndexQueryParserService queryParser() throws IOException {
+        return this.queryParser;
+    }
+
+    @Test
+    public void testDateRangeFilterFormat() throws IOException {
+        IndexQueryParserService queryParser = queryParser();
+        String query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_filter_format.json");
+        queryParser.parse(query).query();
+        // Sadly from NoCacheFilter, we can not access to the delegate filter so we can not check
+        // it's the one we are expecting
+
+        // Test Invalid format
+        query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_filter_format_invalid.json");
+        try {
+            queryParser.parse(query).query();
+            fail("A Range Filter with a specific format but with an unexpected date should raise a QueryParsingException");
+        } catch (QueryParsingException e) {
+            // We expect it
+        }
+    }
+
+    @Test
+    public void testDateRangeQueryFormat() throws IOException {
+        IndexQueryParserService queryParser = queryParser();
+        // We test 01/01/2012 from gte and 2030 for lt
+        String query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_query_format.json");
+        Query parsedQuery = queryParser.parse(query).query();
+        assertThat(parsedQuery, instanceOf(NumericRangeQuery.class));
+
+        // Min value was 01/01/2012 (dd/MM/yyyy)
+        DateTime min = DateTime.parse("2012-01-01T00:00:00.000+00");
+        assertThat(((NumericRangeQuery) parsedQuery).getMin().longValue(), is(min.getMillis()));
+
+        // Max value was 2030 (yyyy)
+        DateTime max = DateTime.parse("2030-01-01T00:00:00.000+00");
+        assertThat(((NumericRangeQuery) parsedQuery).getMax().longValue(), is(max.getMillis()));
+
+        // Test Invalid format
+        query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_query_format_invalid.json");
+        try {
+            queryParser.parse(query).query();
+            fail("A Range Query with a specific format but with an unexpected date should raise a QueryParsingException");
+        } catch (QueryParsingException e) {
+            // We expect it
+        }
+    }
+}

src/test/java/org/elasticsearch/index/query/date_range_filter_format.json

@@ -0,0 +1,13 @@
+{
+    "constant_score": {
+        "filter": {
+            "range" : {
+                "born" : {
+                    "gte": "01/01/2012",
+                    "lt": "2030",
+                    "format": "dd/MM/yyyy||yyyy"
+                }
+            }
+        }
+    }
+}