Skip to content

Adding support for composite index #263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jul 11, 2025
Merged

Adding support for composite index #263

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
999e87c
Adding support for composite index
bharath-techie Jul 7, 2025
361acb6
fetching default codec
bharath-techie Jul 7, 2025
30e5f34
Addressing comment
bharath-techie Jul 9, 2025
11a5206
addressing comment
bharath-techie Jul 10, 2025
5fdb203
Switch main to 3.2.0-SNAPSHOT
reta Jul 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
package org.opensearch.index.codec.customcodecs;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
import org.opensearch.index.codec.composite.composite101.Composite101Codec;
import org.opensearch.index.mapper.MapperService;

import java.util.Set;
Expand Down Expand Up @@ -98,10 +100,18 @@ public Lucene101CustomCodec(Mode mode, int compressionLevel) {
* @param logger The logger.
*/
public Lucene101CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) {
super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger));
super(mode.getCodec(), getDelegateCodec(mapperService, logger));
this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel);
}

private static Codec getDelegateCodec(MapperService mapperService, Logger logger) {
if (mapperService.isCompositeIndexPresent()) {
return new Composite101Codec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger);
} else {
return new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger);
}
}

@Override
public StoredFieldsFormat storedFieldsFormat() {
return storedFieldsFormat;
Expand Down
35 changes: 35 additions & 0 deletions src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import org.opensearch.index.codec.CodecServiceConfig;
import org.opensearch.index.codec.CodecServiceFactory;
import org.opensearch.index.codec.CodecSettings;
import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat;
import org.opensearch.index.engine.EngineConfig;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.index.similarity.SimilarityService;
Expand All @@ -65,6 +66,8 @@
import java.util.Collections;
import java.util.Optional;

import org.mockito.Mockito;

import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_DEFLATE_CODEC;
import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_LZ4_CODEC;
import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_ZSTD_CODEC;
Expand All @@ -90,13 +93,31 @@ public void testZstd() throws Exception {
assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
}

public void testZstdWithCompositeIndex() throws Exception {
Codec codec = createCodecService(false, true).codec("zstd");
assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD, codec);
Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat();
assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
// assert docValues to be of compositeCodec's docValuesFormat
assert codec.docValuesFormat() instanceof Composite912DocValuesFormat;
Copy link
Contributor Author

@bharath-techie bharath-techie Jul 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filter codec does not expose delegate, so could not find a better way for assertion.

}

public void testZstdNoDict() throws Exception {
Codec codec = createCodecService(false).codec("zstd_no_dict");
assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec);
Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat();
assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
}

public void testZstdNoDictWithCompositeIndex() throws Exception {
Codec codec = createCodecService(false, true).codec("zstd_no_dict");
assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec);
Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat();
assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
// assert docValues to be of compositeCodec's docValuesFormat
assert codec.docValuesFormat() instanceof Composite912DocValuesFormat;
}

public void testZstdDeprecatedCodec() {
final IllegalArgumentException e = expectThrows(
IllegalArgumentException.class,
Expand Down Expand Up @@ -224,13 +245,27 @@ private void assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode expec
}

private CodecService createCodecService(boolean isMapperServiceNull) throws IOException {
return createCodecService(isMapperServiceNull, false);
}

private CodecService createCodecService(boolean isMapperServiceNull, boolean isCompositeIndexPresent) throws IOException {
Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
if (isMapperServiceNull) {
return new CustomCodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test"));
}
if (isCompositeIndexPresent) {
return buildCodecServiceWithCompositeIndex(nodeSettings);
}
return buildCodecService(nodeSettings);
}

private CodecService buildCodecServiceWithCompositeIndex(Settings nodeSettings) throws IOException {
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
MapperService service = Mockito.mock(MapperService.class);
Mockito.when(service.isCompositeIndexPresent()).thenReturn(true);
return new CustomCodecService(service, indexSettings, LogManager.getLogger("test"));
}

private CodecService createCodecService(int randomCompressionLevel, String codec) throws IOException {
Settings nodeSettings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
Expand Down
Loading