diff --git a/build.gradle b/build.gradle index 52535c1..50f80b7 100644 --- a/build.gradle +++ b/build.gradle @@ -31,7 +31,7 @@ buildscript { ext { opensearch_group = "org.opensearch" - opensearch_version = System.getProperty("opensearch.version", "3.1.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.2.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index 3d502c7..446bcf0 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -69,18 +69,28 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin ); } } else { - codecs.put(ZSTD_CODEC, new Zstd101Codec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd101Codec(compressionLevel, this::defaultCodec)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(compressionLevel, this::defaultCodec)); if (QatZipperFactory.isQatAvailable()) { - codecs.put(QAT_LZ4_CODEC, new QatLz4101Codec(mapperService, logger, compressionLevel, () -> { - return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); - })); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(mapperService, logger, compressionLevel, () -> { - return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); - })); - codecs.put(QAT_ZSTD_CODEC, new QatZstd101Codec(mapperService, logger, compressionLevel, () -> { + codecs.put( + QAT_LZ4_CODEC, + new QatLz4101Codec( + compressionLevel, + () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }, + this::defaultCodec + ) + ); + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); - })); + }, this::defaultCodec)); + codecs.put( + QAT_ZSTD_CODEC, + new QatZstd101Codec( + compressionLevel, + () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }, + this::defaultCodec + ) + ); } } this.codecs = codecs.immutableMap(); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java index 6c7f4fe..b49fe63 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java @@ -8,14 +8,13 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.lucene101.Lucene101Codec; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; import java.util.Set; +import java.util.function.Supplier; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; @@ -94,11 +93,10 @@ public Lucene101CustomCodec(Mode mode, int compressionLevel) { * * @param mode The compression codec (ZSTD or ZSTDNODICT). * @param compressionLevel The compression level. - * @param mapperService The mapper service. - * @param logger The logger. + * @param defaultCodecSupplier Default OpenSearch codec supplier */ - public Lucene101CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); + public Lucene101CustomCodec(Mode mode, int compressionLevel, Supplier defaultCodecSupplier) { + super(mode.getCodec(), defaultCodecSupplier.get()); this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java index c5ab6d3..23ff319 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java @@ -8,12 +8,10 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.lucene101.Lucene101Codec; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; import java.util.Set; import java.util.function.Supplier; @@ -105,12 +103,11 @@ public Lucene101QatCodec(Mode mode, int compressionLevel, Supplier defaultCodecSupplier, int compressionLevel) { + super(mode.getCodec(), defaultCodecSupplier.get()); this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel); } @@ -121,18 +118,11 @@ public Lucene101QatCodec(Mode mode, int compressionLevel, MapperService mapperSe * * @param mode The compression codec (QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD). * @param compressionLevel The compression level. - * @param mapperService The mapper service. - * @param logger The logger. * @param supplier supplier for QAT mode. + * @param defaultCodecSupplier default opensearch codec supplier */ - public Lucene101QatCodec( - Mode mode, - int compressionLevel, - MapperService mapperService, - Logger logger, - Supplier supplier - ) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); + public Lucene101QatCodec(Mode mode, int compressionLevel, Supplier supplier, Supplier defaultCodecSupplier) { + super(mode.getCodec(), defaultCodecSupplier.get()); this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel, supplier); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java index 8d36d37..48ca918 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java @@ -8,12 +8,11 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; import java.util.function.Supplier; @@ -55,24 +54,22 @@ public QatDeflate101Codec(int compressionLevel, Supplier supplie /** * Creates a new QatDeflate101Codec instance. * - * @param mapperService The mapper service. - * @param logger The logger. + * @param defaultCodecSupplier default opensearch codec supplier * @param compressionLevel The compression level. */ - public QatDeflate101Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger); + public QatDeflate101Codec(Supplier defaultCodecSupplier, int compressionLevel) { + super(Mode.QAT_DEFLATE, defaultCodecSupplier, compressionLevel); } /** * Creates a new QatDeflate101Codec instance. * - * @param mapperService The mapper service. - * @param logger The logger. * @param compressionLevel The compression level. - * @param supplier supplier for QAT acceleration mode. + * @param supplier supplier for QAT acceleration mode + * @param defaultCodecSupplier default opensearch codec supplier */ - public QatDeflate101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { - super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger, supplier); + public QatDeflate101Codec(int compressionLevel, Supplier supplier, Supplier defaultCodecSupplier) { + super(Mode.QAT_DEFLATE, compressionLevel, supplier, defaultCodecSupplier); } /** The name for this codec. */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java index 0f988e0..294c7a3 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java @@ -8,12 +8,11 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; import java.util.function.Supplier; @@ -55,24 +54,22 @@ public QatLz4101Codec(int compressionLevel, Supplier supplier) { /** * Creates a new QatLz4101Codec instance. * - * @param mapperService The mapper service. - * @param logger The logger. + * @param defaultCodecSupplier default opensearch codec supplier * @param compressionLevel The compression level. */ - public QatLz4101Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.QAT_LZ4, compressionLevel, mapperService, logger); + public QatLz4101Codec(Supplier defaultCodecSupplier, int compressionLevel) { + super(Mode.QAT_LZ4, defaultCodecSupplier, compressionLevel); } /** * Creates a new QatLz4101Codec instance. * - * @param mapperService The mapper service. - * @param logger The logger. * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. + * @param defaultCodecSupplier default opensearch codec supplier */ - public QatLz4101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { - super(Mode.QAT_LZ4, compressionLevel, mapperService, logger, supplier); + public QatLz4101Codec(int compressionLevel, Supplier supplier, Supplier defaultCodecSupplier) { + super(Mode.QAT_LZ4, compressionLevel, supplier, defaultCodecSupplier); } /** The name for this codec. */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd101Codec.java index 2153b05..743f6d2 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd101Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd101Codec.java @@ -8,12 +8,11 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; import java.util.function.Supplier; @@ -55,24 +54,22 @@ public QatZstd101Codec(int compressionLevel, Supplier supplier) /** * Creates a new QatZstd101Codec instance. * - * @param mapperService The mapper service. - * @param logger The logger. + * @param defaultCodecSupplier default opensearch codec supplier * @param compressionLevel The compression level. */ - public QatZstd101Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.QAT_ZSTD, compressionLevel, mapperService, logger); + public QatZstd101Codec(Supplier defaultCodecSupplier, int compressionLevel) { + super(Mode.QAT_ZSTD, defaultCodecSupplier, compressionLevel); } /** * Creates a new QatZstd101Codec instance. * - * @param mapperService The mapper service. - * @param logger The logger. * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. + * @param defaultCodecSupplier default opensearch codec supplier */ - public QatZstd101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { - super(Mode.QAT_ZSTD, compressionLevel, mapperService, logger, supplier); + public QatZstd101Codec(int compressionLevel, Supplier supplier, Supplier defaultCodecSupplier) { + super(Mode.QAT_ZSTD, compressionLevel, supplier, defaultCodecSupplier); } /** The name for this codec. */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java index c0ebc0e..b61b7e6 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java @@ -8,14 +8,14 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; +import java.util.function.Supplier; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; @@ -42,12 +42,11 @@ public Zstd101Codec(int compressionLevel) { /** * Creates a new ZstdCodec instance. * - * @param mapperService The mapper service. - * @param logger The logger. * @param compressionLevel The compression level. + * @param defaultCodecSupplier default opensearch codec supplier */ - public Zstd101Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD, compressionLevel, mapperService, logger); + public Zstd101Codec(int compressionLevel, Supplier defaultCodecSupplier) { + super(Mode.ZSTD, compressionLevel, defaultCodecSupplier); } /** The name for this codec. */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java index ea4ee5c..555e0bf 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java @@ -8,14 +8,14 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; +import java.util.function.Supplier; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; @@ -39,12 +39,11 @@ public ZstdNoDict101Codec(int compressionLevel) { /** * Creates a new ZstdNoDictCodec instance. * - * @param mapperService The mapper service. - * @param logger The logger. * @param compressionLevel The compression level. + * @param defaultCodecSupplier default opensearch codec supplier */ - public ZstdNoDict101Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); + public ZstdNoDict101Codec(int compressionLevel, Supplier defaultCodecSupplier) { + super(Mode.ZSTD_NO_DICT, compressionLevel, defaultCodecSupplier); } /** The name for this codec. */ diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index bc9d0db..7804ab6 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -52,6 +52,7 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; @@ -65,6 +66,8 @@ import java.util.Collections; import java.util.Optional; +import org.mockito.Mockito; + import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_DEFLATE_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_LZ4_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_ZSTD_CODEC; @@ -90,6 +93,15 @@ public void testZstd() throws Exception { assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } + public void testZstdWithCompositeIndex() throws Exception { + Codec codec = createCodecService(false, true).codec("zstd"); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + // assert docValues to be of compositeCodec's docValuesFormat + assert codec.docValuesFormat() instanceof Composite912DocValuesFormat; + } + public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec); @@ -97,6 +109,15 @@ public void testZstdNoDict() throws Exception { assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } + public void testZstdNoDictWithCompositeIndex() throws Exception { + Codec codec = createCodecService(false, true).codec("zstd_no_dict"); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + // assert docValues to be of compositeCodec's docValuesFormat + assert codec.docValuesFormat() instanceof Composite912DocValuesFormat; + } + public void testZstdDeprecatedCodec() { final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, @@ -224,13 +245,27 @@ private void assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode expec } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { + return createCodecService(isMapperServiceNull, false); + } + + private CodecService createCodecService(boolean isMapperServiceNull, boolean isCompositeIndexPresent) throws IOException { Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); if (isMapperServiceNull) { return new CustomCodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test")); } + if (isCompositeIndexPresent) { + return buildCodecServiceWithCompositeIndex(nodeSettings); + } return buildCodecService(nodeSettings); } + private CodecService buildCodecServiceWithCompositeIndex(Settings nodeSettings) throws IOException { + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); + MapperService service = Mockito.mock(MapperService.class); + Mockito.when(service.isCompositeIndexPresent()).thenReturn(true); + return new CustomCodecService(service, indexSettings, LogManager.getLogger("test")); + } + private CodecService createCodecService(int randomCompressionLevel, String codec) throws IOException { Settings nodeSettings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())