Skip to content

Commit 8b23e15

Browse files
night-owl-1709night-owl-1709
committed
Added code changes to support field level stat on segment
Signed-off-by: Abhinav Tripathi <[email protected]>
1 parent 696ed65 commit 8b23e15

27 files changed

+1102
-56
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
55

66
## [Unreleased 3.x]
77
### Added
8+
- Add field-level statistics to the _stats API for Lucene segment file sizes ([#12113](https://github.com/opensearch-project/OpenSearch/issues/12113))
89
- [Feature Request] Enhance Terms lookup query to support query clause instead of docId ([#18195](https://github.com/opensearch-project/OpenSearch/issues/18195))
910
- Add hierarchical routing processors for ingest and search pipelines ([#18826](https://github.com/opensearch-project/OpenSearch/pull/18826))
1011
- Add ACL-aware routing processors for ingest and search pipelines ([#18834](https://github.com/opensearch-project/OpenSearch/pull/18834))

‎rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,11 @@
123123
"description":"Whether to report the aggregated disk usage of each one of the Lucene index files (only applies if segment stats are requested)",
124124
"default":false
125125
},
126+
"include_field_level_segment_file_sizes":{
127+
"type":"boolean",
128+
"description":"Whether to report the disk usage of Lucene index files at the field level (only applies if segment stats are requested)",
129+
"default":false
130+
},
126131
"include_unloaded_segments":{
127132
"type":"boolean",
128133
"description":"If set to true segment stats will include stats for segments that are not currently loaded into memory",

‎rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,11 @@
222222
"type":"boolean",
223223
"description":"Whether to report the aggregated disk usage of each one of the Lucene index files (only applies if segment stats are requested)",
224224
"default":false
225+
},
226+
"include_field_level_segment_file_sizes":{
227+
"type":"boolean",
228+
"description":"Whether to report the disk usage of Lucene index files at the field level (only applies if segment stats are requested)",
229+
"default":false
225230
}
226231
}
227232
}

‎server/build.gradle‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -395,17 +395,20 @@ tasks.named("sourcesJar").configure {
395395
tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) {
396396
logger.info("Comparing public APIs from ${version} to ${japicmpCompareTarget}")
397397
// See please https://github.com/siom79/japicmp/issues/201
398-
compatibilityChangeExcludes = [ "METHOD_ABSTRACT_NOW_DEFAULT", "METHOD_ADDED_TO_INTERFACE" ]
398+
compatibilityChangeExcludes = [ "METHOD_ABSTRACT_NOW_DEFAULT", "METHOD_ADDED_TO_INTERFACE", "METHOD_REMOVED_IN_SUPERCLASS" ]
399399
oldClasspath.from(files("${buildDir}/japicmp-target/opensearch-${japicmpCompareTarget}.jar"))
400400
newClasspath.from(tasks.named('jar'))
401+
// Restrict to modified elements only and fail only on binary-incompatible changes
401402
onlyModified = true
403+
onlyBinaryIncompatibleModified = true
402404
failOnModification = true
403405
ignoreMissingClasses = true
404406
failOnSourceIncompatibility = true
405407
annotationIncludes = ['@org.opensearch.common.annotation.PublicApi', '@org.opensearch.common.annotation.DeprecatedApi']
406408
annotationExcludes = ['@org.opensearch.common.annotation.InternalApi', '@org.opensearch.common.annotation.ExperimentalApi']
407409
txtOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.txt")
408410
htmlOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.html")
411+
xmlOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.xml")
409412
dependsOn downloadJapicmpCompareTarget
410413
}
411414

‎server/src/internalClusterTest/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerIT.java‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -86,15 +86,15 @@ public void testAutoForceMergeFeatureFlagDisabled() throws InterruptedException,
8686
assertNotNull(shard);
8787

8888
// Before stats
89-
SegmentsStats segmentsStatsBefore = shard.segmentStats(false, false);
89+
SegmentsStats segmentsStatsBefore = shard.segmentStats(false, false, false);
9090

9191
// This is to make sure auto force merge action gets triggered multiple times ang gets successful at least once.
9292
Thread.sleep(TimeValue.parseTimeValue(SCHEDULER_INTERVAL, "test").getMillis() * 3);
9393
// refresh to clear old segments
9494
flushAndRefresh(INDEX_NAME_1);
9595

9696
// After stats
97-
SegmentsStats segmentsStatsAfter = shard.segmentStats(false, false);
97+
SegmentsStats segmentsStatsAfter = shard.segmentStats(false, false, false);
9898
assertEquals(segmentsStatsBefore.getCount(), segmentsStatsAfter.getCount());
9999

100100
// Deleting the index (so that ref count drops to zero for all the files) and then pruning the cache to clear it to avoid any file
@@ -123,9 +123,9 @@ public void testAutoForceMergeTriggeringWithOneShardOfNonWarmCandidate() throws
123123
}
124124
IndexShard shard = getIndexShard(dataNode, INDEX_NAME_1);
125125
assertNotNull(shard);
126-
SegmentsStats segmentsStatsBefore = shard.segmentStats(false, false);
126+
SegmentsStats segmentsStatsBefore = shard.segmentStats(false, false, false);
127127
Thread.sleep(TimeValue.parseTimeValue(SCHEDULER_INTERVAL, "test").getMillis() * 3);
128-
SegmentsStats segmentsStatsAfter = shard.segmentStats(false, false);
128+
SegmentsStats segmentsStatsAfter = shard.segmentStats(false, false, false);
129129
assertEquals(segmentsStatsBefore.getCount(), segmentsStatsAfter.getCount());
130130
assertAcked(client().admin().indices().prepareDelete(INDEX_NAME_1).get());
131131
}
@@ -150,9 +150,9 @@ public void testAutoForceMergeTriggeringBasicWithOneShard() throws Exception {
150150
}
151151
IndexShard shard = getIndexShard(dataNode, INDEX_NAME_1);
152152
assertNotNull(shard);
153-
SegmentsStats segmentsStatsBefore = shard.segmentStats(false, false);
154-
waitUntil(() -> shard.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
155-
SegmentsStats segmentsStatsAfter = shard.segmentStats(false, false);
153+
SegmentsStats segmentsStatsBefore = shard.segmentStats(false, false, false);
154+
waitUntil(() -> shard.segmentStats(false, false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
155+
SegmentsStats segmentsStatsAfter = shard.segmentStats(false, false, false);
156156
assertTrue((int) segmentsStatsBefore.getCount() > segmentsStatsAfter.getCount());
157157
assertEquals((int) SEGMENT_COUNT, segmentsStatsAfter.getCount());
158158
assertAcked(client().admin().indices().prepareDelete(INDEX_NAME_1).get());
@@ -211,26 +211,26 @@ public void testAutoForceMergeTriggeringBasicWithFiveShardsOfTwoIndex() throws E
211211
assertNotNull(shard4);
212212
assertNotNull(shard5);
213213

214-
SegmentsStats segmentsStatsForShard1Before = shard1.segmentStats(false, false);
215-
SegmentsStats segmentsStatsForShard2Before = shard2.segmentStats(false, false);
216-
SegmentsStats segmentsStatsForShard3Before = shard3.segmentStats(false, false);
217-
SegmentsStats segmentsStatsForShard4Before = shard4.segmentStats(false, false);
218-
SegmentsStats segmentsStatsForShard5Before = shard5.segmentStats(false, false);
214+
SegmentsStats segmentsStatsForShard1Before = shard1.segmentStats(false, false, false);
215+
SegmentsStats segmentsStatsForShard2Before = shard2.segmentStats(false, false, false);
216+
SegmentsStats segmentsStatsForShard3Before = shard3.segmentStats(false, false, false);
217+
SegmentsStats segmentsStatsForShard4Before = shard4.segmentStats(false, false, false);
218+
SegmentsStats segmentsStatsForShard5Before = shard5.segmentStats(false, false, false);
219219
AtomicLong totalSegmentsBefore = new AtomicLong(
220220
segmentsStatsForShard1Before.getCount() + segmentsStatsForShard2Before.getCount() + segmentsStatsForShard3Before.getCount()
221221
+ segmentsStatsForShard4Before.getCount() + segmentsStatsForShard5Before.getCount()
222222
);
223223
assertTrue(totalSegmentsBefore.get() > 5);
224-
waitUntil(() -> shard1.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
225-
waitUntil(() -> shard2.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
226-
waitUntil(() -> shard3.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
227-
waitUntil(() -> shard4.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
228-
waitUntil(() -> shard5.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
229-
SegmentsStats segmentsStatsForShard1After = shard1.segmentStats(false, false);
230-
SegmentsStats segmentsStatsForShard2After = shard2.segmentStats(false, false);
231-
SegmentsStats segmentsStatsForShard3After = shard3.segmentStats(false, false);
232-
SegmentsStats segmentsStatsForShard4After = shard4.segmentStats(false, false);
233-
SegmentsStats segmentsStatsForShard5After = shard5.segmentStats(false, false);
224+
waitUntil(() -> shard1.segmentStats(false, false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
225+
waitUntil(() -> shard2.segmentStats(false, false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
226+
waitUntil(() -> shard3.segmentStats(false, false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
227+
waitUntil(() -> shard4.segmentStats(false, false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
228+
waitUntil(() -> shard5.segmentStats(false, false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES);
229+
SegmentsStats segmentsStatsForShard1After = shard1.segmentStats(false, false, false);
230+
SegmentsStats segmentsStatsForShard2After = shard2.segmentStats(false, false, false);
231+
SegmentsStats segmentsStatsForShard3After = shard3.segmentStats(false, false, false);
232+
SegmentsStats segmentsStatsForShard4After = shard4.segmentStats(false, false, false);
233+
SegmentsStats segmentsStatsForShard5After = shard5.segmentStats(false, false, false);
234234
AtomicLong totalSegmentsAfter = new AtomicLong(
235235
segmentsStatsForShard1After.getCount() + segmentsStatsForShard2After.getCount() + segmentsStatsForShard3After.getCount()
236236
+ segmentsStatsForShard4After.getCount() + segmentsStatsForShard5After.getCount()

‎server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,11 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C
227227
completion = indexShard.completionStats(flags.completionDataFields());
228228
break;
229229
case Segments:
230-
segments = indexShard.segmentStats(flags.includeSegmentFileSizes(), flags.includeUnloadedSegments());
230+
segments = indexShard.segmentStats(
231+
flags.includeSegmentFileSizes(),
232+
flags.includeFieldLevelSegmentFileSizes(),
233+
flags.includeUnloadedSegments()
234+
);
231235
break;
232236
case Translog:
233237
translog = indexShard.translogStats();

‎server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ public class CommonStatsFlags implements Writeable, Cloneable {
6161
private String[] fieldDataFields = null;
6262
private String[] completionDataFields = null;
6363
private boolean includeSegmentFileSizes = false;
64+
private boolean includeFieldLevelSegmentFileSizes = false;
6465
private boolean includeUnloadedSegments = false;
6566
private boolean includeAllShardIndexingPressureTrackers = false;
6667
private boolean includeOnlyTopIndexingPressureMetrics = false;
@@ -94,6 +95,11 @@ public CommonStatsFlags(StreamInput in) throws IOException {
9495
fieldDataFields = in.readStringArray();
9596
completionDataFields = in.readStringArray();
9697
includeSegmentFileSizes = in.readBoolean();
98+
if (in.getVersion().onOrAfter(Version.CURRENT)) {
99+
includeFieldLevelSegmentFileSizes = in.readBoolean();
100+
} else {
101+
includeFieldLevelSegmentFileSizes = false;
102+
}
97103
includeUnloadedSegments = in.readBoolean();
98104
includeAllShardIndexingPressureTrackers = in.readBoolean();
99105
includeOnlyTopIndexingPressureMetrics = in.readBoolean();
@@ -121,6 +127,9 @@ public void writeTo(StreamOutput out) throws IOException {
121127
out.writeStringArrayNullable(fieldDataFields);
122128
out.writeStringArrayNullable(completionDataFields);
123129
out.writeBoolean(includeSegmentFileSizes);
130+
if (out.getVersion().onOrAfter(Version.CURRENT)) {
131+
out.writeBoolean(includeFieldLevelSegmentFileSizes);
132+
}
124133
out.writeBoolean(includeUnloadedSegments);
125134
out.writeBoolean(includeAllShardIndexingPressureTrackers);
126135
out.writeBoolean(includeOnlyTopIndexingPressureMetrics);
@@ -142,6 +151,7 @@ public CommonStatsFlags all() {
142151
fieldDataFields = null;
143152
completionDataFields = null;
144153
includeSegmentFileSizes = false;
154+
includeFieldLevelSegmentFileSizes = false;
145155
includeUnloadedSegments = false;
146156
includeAllShardIndexingPressureTrackers = false;
147157
includeOnlyTopIndexingPressureMetrics = false;
@@ -159,6 +169,7 @@ public CommonStatsFlags clear() {
159169
fieldDataFields = null;
160170
completionDataFields = null;
161171
includeSegmentFileSizes = false;
172+
includeFieldLevelSegmentFileSizes = false;
162173
includeUnloadedSegments = false;
163174
includeAllShardIndexingPressureTrackers = false;
164175
includeOnlyTopIndexingPressureMetrics = false;
@@ -223,6 +234,11 @@ public CommonStatsFlags includeSegmentFileSizes(boolean includeSegmentFileSizes)
223234
return this;
224235
}
225236

237+
public CommonStatsFlags includeFieldLevelSegmentFileSizes(boolean includeFieldLevelSegmentFileSizes) {
238+
this.includeFieldLevelSegmentFileSizes = includeFieldLevelSegmentFileSizes;
239+
return this;
240+
}
241+
226242
public CommonStatsFlags includeUnloadedSegments(boolean includeUnloadedSegments) {
227243
this.includeUnloadedSegments = includeUnloadedSegments;
228244
return this;
@@ -269,6 +285,10 @@ public boolean includeSegmentFileSizes() {
269285
return this.includeSegmentFileSizes;
270286
}
271287

288+
public boolean includeFieldLevelSegmentFileSizes() {
289+
return this.includeFieldLevelSegmentFileSizes;
290+
}
291+
272292
public void setIncludeIndicesStatsByLevel(boolean includeIndicesStatsByLevel) {
273293
this.includeIndicesStatsByLevel = includeIndicesStatsByLevel;
274294
}

‎server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,15 @@ public IndicesStatsRequest includeSegmentFileSizes(boolean includeSegmentFileSiz
292292
return this;
293293
}
294294

295+
public boolean includeFieldLevelSegmentFileSizes() {
296+
return flags.includeFieldLevelSegmentFileSizes();
297+
}
298+
299+
public IndicesStatsRequest includeFieldLevelSegmentFileSizes(boolean includeFieldLevelSegmentFileSizes) {
300+
flags.includeFieldLevelSegmentFileSizes(includeFieldLevelSegmentFileSizes);
301+
return this;
302+
}
303+
295304
public IndicesStatsRequest includeUnloadedSegments(boolean includeUnloadedSegments) {
296305
flags.includeUnloadedSegments(includeUnloadedSegments);
297306
return this;

‎server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,4 +181,9 @@ public IndicesStatsRequestBuilder setIncludeSegmentFileSizes(boolean includeSegm
181181
request.includeSegmentFileSizes(includeSegmentFileSizes);
182182
return this;
183183
}
184+
185+
public IndicesStatsRequestBuilder setIncludeFieldLevelSegmentFileSizes(boolean includeFieldLevelSegmentFileSizes) {
186+
request.includeFieldLevelSegmentFileSizes(includeFieldLevelSegmentFileSizes);
187+
return this;
188+
}
184189
}

‎server/src/main/java/org/opensearch/index/engine/Engine.java‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -923,14 +923,18 @@ public final CommitStats commitStats() {
923923
/**
924924
* Global stats on segments.
925925
*/
926-
public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
926+
public SegmentsStats segmentsStats(
927+
boolean includeSegmentFileSizes,
928+
boolean includeFieldLevelSegmentFileSizes,
929+
boolean includeUnloadedSegments
930+
) {
927931
ensureOpen();
928932
Set<String> segmentName = new HashSet<>();
929933
SegmentsStats stats = new SegmentsStats();
930934
try (Searcher searcher = acquireSearcher("segments_stats", SearcherScope.INTERNAL)) {
931935
for (LeafReaderContext ctx : searcher.getIndexReader().getContext().leaves()) {
932936
SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
933-
fillSegmentStats(segmentReader, includeSegmentFileSizes, stats);
937+
fillSegmentStats(segmentReader, includeSegmentFileSizes, includeFieldLevelSegmentFileSizes, stats);
934938
segmentName.add(segmentReader.getSegmentName());
935939
}
936940
}
@@ -939,14 +943,22 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
939943
for (LeafReaderContext ctx : searcher.getIndexReader().getContext().leaves()) {
940944
SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
941945
if (segmentName.contains(segmentReader.getSegmentName()) == false) {
942-
fillSegmentStats(segmentReader, includeSegmentFileSizes, stats);
946+
fillSegmentStats(segmentReader, includeSegmentFileSizes, includeFieldLevelSegmentFileSizes, stats);
943947
}
944948
}
945949
}
946950
writerSegmentStats(stats);
947951
return stats;
948952
}
949953

954+
/**
955+
* Backwards-compatible overload retained for binary compatibility.
956+
* Delegates to the new method with field-level stats disabled by default.
957+
*/
958+
public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
959+
return segmentsStats(includeSegmentFileSizes, false, includeUnloadedSegments);
960+
}
961+
950962
/**
951963
* @return Stats for pull-based ingestion.
952964
*/
@@ -970,12 +982,33 @@ protected TranslogDeletionPolicy getTranslogDeletionPolicy(EngineConfig engineCo
970982
);
971983
}
972984

973-
protected void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) {
985+
protected void fillSegmentStats(
986+
SegmentReader segmentReader,
987+
boolean includeSegmentFileSizes,
988+
boolean includeFieldLevelSegmentFileSizes,
989+
SegmentsStats stats
990+
) {
974991
stats.add(1);
975992
if (includeSegmentFileSizes) {
976993
// TODO: consider moving this to StoreStats
977994
stats.addFileSizes(getSegmentFileSizes(segmentReader));
978995
}
996+
if (includeFieldLevelSegmentFileSizes) {
997+
try {
998+
FieldLevelSegmentStatsCalculator calculator = new FieldLevelSegmentStatsCalculator();
999+
Map<String, Map<String, Long>> fieldLevelStats = calculator.calculateFieldLevelStats(segmentReader);
1000+
stats.addFieldLevelFileSizes(fieldLevelStats);
1001+
} catch (Exception e) {
1002+
logger.warn(
1003+
() -> new ParameterizedMessage(
1004+
"Failed to calculate field-level segment statistics for segment [{}]",
1005+
segmentReader.getSegmentName()
1006+
),
1007+
e
1008+
);
1009+
// Continue without field-level stats rather than failing the entire request
1010+
}
1011+
}
9791012
}
9801013

9811014
boolean shouldCleanupUnreferencedFiles() {
@@ -1162,7 +1195,7 @@ public boolean refreshNeeded() {
11621195
return searcher.getDirectoryReader().isCurrent() == false;
11631196
}
11641197
} catch (IOException e) {
1165-
logger.error("failed to access searcher manager", e);
1198+
logger.error(() -> new ParameterizedMessage("failed to access searcher manager"), e);
11661199
failEngine("failed to access searcher manager", e);
11671200
throw new EngineException(shardId, "failed to access searcher manager", e);
11681201
} finally {

0 commit comments

Comments
 (0)