Addressed comments

Signed-off-by: Sriram Ganesh <[email protected]>

Author: rgsriram

server/src/main/java/org/opensearch/index/AdaptiveTieredMergePolicyProvider.java

@@ -41,7 +41,7 @@ public class AdaptiveTieredMergePolicyProvider implements MergePolicyProvider {
     private static final ByteSizeValue SMALL_SHARD_MAX_SEGMENT = new ByteSizeValue(50, ByteSizeUnit.MB);
     private static final ByteSizeValue MEDIUM_SHARD_MAX_SEGMENT = new ByteSizeValue(200, ByteSizeUnit.MB);
     private static final ByteSizeValue LARGE_SHARD_MAX_SEGMENT = new ByteSizeValue(1, ByteSizeUnit.GB);
-    private static final ByteSizeValue VERY_LARGE_SHARD_MAX_SEGMENT = new ByteSizeValue(2, ByteSizeUnit.GB);
+    private static final ByteSizeValue VERY_LARGE_SHARD_MAX_SEGMENT = new ByteSizeValue(5, ByteSizeUnit.GB);
 
     // Adaptive floor segment sizes
     private static final ByteSizeValue SMALL_SHARD_FLOOR = new ByteSizeValue(10, ByteSizeUnit.MB);
@@ -136,36 +136,12 @@ private ShardSizeCategory categorizeShardSize(long sizeBytes) {
     }
 
     private void applyAdaptiveSettings(ShardSizeCategory category) {
-        ByteSizeValue maxSegmentSize;
-        ByteSizeValue floorSegmentSize;
-        double segmentsPerTier;
-
-        switch (category) {
-            case SMALL:
-                maxSegmentSize = SMALL_SHARD_MAX_SEGMENT;
-                floorSegmentSize = SMALL_SHARD_FLOOR;
-                segmentsPerTier = SMALL_SHARD_SEGMENTS_PER_TIER;
-                break;
-            case MEDIUM:
-                maxSegmentSize = MEDIUM_SHARD_MAX_SEGMENT;
-                floorSegmentSize = MEDIUM_SHARD_FLOOR;
-                segmentsPerTier = MEDIUM_SHARD_SEGMENTS_PER_TIER;
-                break;
-            case LARGE:
-                maxSegmentSize = LARGE_SHARD_MAX_SEGMENT;
-                floorSegmentSize = LARGE_SHARD_FLOOR;
-                segmentsPerTier = LARGE_SHARD_SEGMENTS_PER_TIER;
-                break;
-            case VERY_LARGE:
-                maxSegmentSize = VERY_LARGE_SHARD_MAX_SEGMENT;
-                floorSegmentSize = VERY_LARGE_SHARD_FLOOR;
-                segmentsPerTier = VERY_LARGE_SHARD_SEGMENTS_PER_TIER;
-                break;
-            default:
-                maxSegmentSize = MEDIUM_SHARD_MAX_SEGMENT;
-                floorSegmentSize = MEDIUM_SHARD_FLOOR;
-                segmentsPerTier = MEDIUM_SHARD_SEGMENTS_PER_TIER;
-        }
+        // Use smooth interpolation instead of discrete categories to avoid dramatic parameter jumps
+        long shardSizeBytes = estimateShardSize();
+
+        ByteSizeValue maxSegmentSize = calculateSmoothMaxSegmentSize(shardSizeBytes);
+        ByteSizeValue floorSegmentSize = calculateSmoothFloorSegmentSize(shardSizeBytes);
+        double segmentsPerTier = calculateSmoothSegmentsPerTier(shardSizeBytes);
 
         // Apply the adaptive settings
         tieredMergePolicy.setMaxMergedSegmentMB(maxSegmentSize.getMbFrac());
@@ -186,6 +162,90 @@ private void applyAdaptiveSettings(ShardSizeCategory category) {
         );
     }
 
+    /**
+     * Calculate smooth max segment size using logarithmic interpolation
+     * to avoid dramatic jumps at category boundaries
+     */
+    private ByteSizeValue calculateSmoothMaxSegmentSize(long shardSizeBytes) {
+        // Use logarithmic interpolation between reference points
+        // Reference points: 50MB@100MB, 200MB@1GB, 1GB@10GB, 5GB@100GB
+        double logSize = Math.log10(shardSizeBytes);
+
+        if (logSize < 8.0) { // < 100MB
+            return SMALL_SHARD_MAX_SEGMENT;
+        } else if (logSize < 9.0) { // 100MB - 1GB
+            // Linear interpolation between 50MB and 200MB
+            double ratio = (logSize - 8.0) / 1.0;
+            long interpolatedSize = (long) (SMALL_SHARD_MAX_SEGMENT.getBytes() + ratio * (MEDIUM_SHARD_MAX_SEGMENT.getBytes()
+                - SMALL_SHARD_MAX_SEGMENT.getBytes()));
+            return new ByteSizeValue(interpolatedSize);
+        } else if (logSize < 10.0) { // 1GB - 10GB
+            // Linear interpolation between 200MB and 1GB
+            double ratio = (logSize - 9.0) / 1.0;
+            long interpolatedSize = (long) (MEDIUM_SHARD_MAX_SEGMENT.getBytes() + ratio * (LARGE_SHARD_MAX_SEGMENT.getBytes()
+                - MEDIUM_SHARD_MAX_SEGMENT.getBytes()));
+            return new ByteSizeValue(interpolatedSize);
+        } else if (logSize < 11.0) { // 10GB - 100GB
+            // Linear interpolation between 1GB and 5GB
+            double ratio = (logSize - 10.0) / 1.0;
+            long interpolatedSize = (long) (LARGE_SHARD_MAX_SEGMENT.getBytes() + ratio * (VERY_LARGE_SHARD_MAX_SEGMENT.getBytes()
+                - LARGE_SHARD_MAX_SEGMENT.getBytes()));
+            return new ByteSizeValue(interpolatedSize);
+        } else { // >= 100GB
+            return VERY_LARGE_SHARD_MAX_SEGMENT;
+        }
+    }
+
+    /**
+     * Calculate smooth floor segment size using logarithmic interpolation
+     */
+    private ByteSizeValue calculateSmoothFloorSegmentSize(long shardSizeBytes) {
+        double logSize = Math.log10(shardSizeBytes);
+
+        if (logSize < 8.0) { // < 100MB
+            return SMALL_SHARD_FLOOR;
+        } else if (logSize < 9.0) { // 100MB - 1GB
+            double ratio = (logSize - 8.0) / 1.0;
+            long interpolatedSize = (long) (SMALL_SHARD_FLOOR.getBytes() + ratio * (MEDIUM_SHARD_FLOOR.getBytes() - SMALL_SHARD_FLOOR
+                .getBytes()));
+            return new ByteSizeValue(interpolatedSize);
+        } else if (logSize < 10.0) { // 1GB - 10GB
+            double ratio = (logSize - 9.0) / 1.0;
+            long interpolatedSize = (long) (MEDIUM_SHARD_FLOOR.getBytes() + ratio * (LARGE_SHARD_FLOOR.getBytes() - MEDIUM_SHARD_FLOOR
+                .getBytes()));
+            return new ByteSizeValue(interpolatedSize);
+        } else if (logSize < 11.0) { // 10GB - 100GB
+            double ratio = (logSize - 10.0) / 1.0;
+            long interpolatedSize = (long) (LARGE_SHARD_FLOOR.getBytes() + ratio * (VERY_LARGE_SHARD_FLOOR.getBytes() - LARGE_SHARD_FLOOR
+                .getBytes()));
+            return new ByteSizeValue(interpolatedSize);
+        } else { // >= 100GB
+            return VERY_LARGE_SHARD_FLOOR;
+        }
+    }
+
+    /**
+     * Calculate smooth segments per tier using logarithmic interpolation
+     */
+    private double calculateSmoothSegmentsPerTier(long shardSizeBytes) {
+        double logSize = Math.log10(shardSizeBytes);
+
+        if (logSize < 8.0) { // < 100MB
+            return SMALL_SHARD_SEGMENTS_PER_TIER;
+        } else if (logSize < 9.0) { // 100MB - 1GB
+            double ratio = (logSize - 8.0) / 1.0;
+            return SMALL_SHARD_SEGMENTS_PER_TIER + ratio * (MEDIUM_SHARD_SEGMENTS_PER_TIER - SMALL_SHARD_SEGMENTS_PER_TIER);
+        } else if (logSize < 10.0) { // 1GB - 10GB
+            double ratio = (logSize - 9.0) / 1.0;
+            return MEDIUM_SHARD_SEGMENTS_PER_TIER + ratio * (LARGE_SHARD_SEGMENTS_PER_TIER - MEDIUM_SHARD_SEGMENTS_PER_TIER);
+        } else if (logSize < 11.0) { // 10GB - 100GB
+            double ratio = (logSize - 10.0) / 1.0;
+            return LARGE_SHARD_SEGMENTS_PER_TIER + ratio * (VERY_LARGE_SHARD_SEGMENTS_PER_TIER - LARGE_SHARD_SEGMENTS_PER_TIER);
+        } else { // >= 100GB
+            return VERY_LARGE_SHARD_SEGMENTS_PER_TIER;
+        }
+    }
+
     private void applyDefaultSettings() {
         // Fallback to the original default settings
         tieredMergePolicy.setMaxMergedSegmentMB(5 * 1024); // 5GB

server/src/main/java/org/opensearch/index/analysis/SegmentTopologyAnalyzer.java

@@ -168,45 +168,100 @@ public MergePolicyRecommendations analyzeAndRecommend() {
     }
 
     private long calculateRecommendedMaxSegmentSize() {
-        // Base recommendation on total size and current distribution
-        long totalSize = totalSizeBytes;
+        // Use smooth interpolation to avoid dramatic parameter jumps
+        return calculateSmoothMaxSegmentSize(totalSizeBytes);
+    }
+
+    /**
+     * Calculate smooth max segment size using logarithmic interpolation
+     * to avoid dramatic jumps at category boundaries
+     */
+    private long calculateSmoothMaxSegmentSize(long shardSizeBytes) {
+        // Reference points: 50MB@100MB, 200MB@1GB, 1GB@10GB, 5GB@100GB
+        double logSize = Math.log10(shardSizeBytes);
 
-        if (totalSize < 100 * 1024 * 1024) { // < 100MB
+        if (logSize < 8.0) { // < 100MB
             return 50 * 1024 * 1024; // 50MB
-        } else if (totalSize < 1024 * 1024 * 1024) { // < 1GB
-            return 200 * 1024 * 1024; // 200MB
-        } else if (totalSize < 10L * 1024 * 1024 * 1024) { // < 10GB
-            return 1024 * 1024 * 1024; // 1GB
-        } else { // >= 10GB
-            return 2L * 1024 * 1024 * 1024; // 2GB
+        } else if (logSize < 9.0) { // 100MB - 1GB
+            // Linear interpolation between 50MB and 200MB
+            double ratio = (logSize - 8.0) / 1.0;
+            long smallSize = 50 * 1024 * 1024;
+            long mediumSize = 200 * 1024 * 1024;
+            return (long) (smallSize + ratio * (mediumSize - smallSize));
+        } else if (logSize < 10.0) { // 1GB - 10GB
+            // Linear interpolation between 200MB and 1GB
+            double ratio = (logSize - 9.0) / 1.0;
+            long mediumSize = 200 * 1024 * 1024;
+            long largeSize = 1024 * 1024 * 1024;
+            return (long) (mediumSize + ratio * (largeSize - mediumSize));
+        } else if (logSize < 11.0) { // 10GB - 100GB
+            // Linear interpolation between 1GB and 5GB
+            double ratio = (logSize - 10.0) / 1.0;
+            long largeSize = 1024 * 1024 * 1024;
+            long veryLargeSize = 5L * 1024 * 1024 * 1024;
+            return (long) (largeSize + ratio * (veryLargeSize - largeSize));
+        } else { // >= 100GB
+            return 5L * 1024 * 1024 * 1024; // 5GB
         }
     }
 
     private long calculateRecommendedFloorSegmentSize() {
-        long totalSize = totalSizeBytes;
+        // Use smooth interpolation to avoid dramatic parameter jumps
+        return calculateSmoothFloorSegmentSize(totalSizeBytes);
+    }
 
-        if (totalSize < 100 * 1024 * 1024) { // < 100MB
+    /**
+     * Calculate smooth floor segment size using logarithmic interpolation
+     */
+    private long calculateSmoothFloorSegmentSize(long shardSizeBytes) {
+        double logSize = Math.log10(shardSizeBytes);
+
+        if (logSize < 8.0) { // < 100MB
             return 10 * 1024 * 1024; // 10MB
-        } else if (totalSize < 1024 * 1024 * 1024) { // < 1GB
-            return 25 * 1024 * 1024; // 25MB
-        } else if (totalSize < 10L * 1024 * 1024 * 1024) { // < 10GB
-            return 50 * 1024 * 1024; // 50MB
-        } else { // >= 10GB
+        } else if (logSize < 9.0) { // 100MB - 1GB
+            double ratio = (logSize - 8.0) / 1.0;
+            long smallSize = 10 * 1024 * 1024;
+            long mediumSize = 25 * 1024 * 1024;
+            return (long) (smallSize + ratio * (mediumSize - smallSize));
+        } else if (logSize < 10.0) { // 1GB - 10GB
+            double ratio = (logSize - 9.0) / 1.0;
+            long mediumSize = 25 * 1024 * 1024;
+            long largeSize = 50 * 1024 * 1024;
+            return (long) (mediumSize + ratio * (largeSize - mediumSize));
+        } else if (logSize < 11.0) { // 10GB - 100GB
+            double ratio = (logSize - 10.0) / 1.0;
+            long largeSize = 50 * 1024 * 1024;
+            long veryLargeSize = 100 * 1024 * 1024;
+            return (long) (largeSize + ratio * (veryLargeSize - largeSize));
+        } else { // >= 100GB
             return 100 * 1024 * 1024; // 100MB
         }
     }
 
     private int calculateOptimalSegmentCount() {
-        long totalSize = totalSizeBytes;
+        // Use smooth interpolation to avoid dramatic parameter jumps
+        return (int) Math.round(calculateSmoothSegmentsPerTier(totalSizeBytes));
+    }
 
-        if (totalSize < 100 * 1024 * 1024) { // < 100MB
-            return 5;
-        } else if (totalSize < 1024 * 1024 * 1024) { // < 1GB
-            return 8;
-        } else if (totalSize < 10L * 1024 * 1024 * 1024) { // < 10GB
-            return 10;
-        } else { // >= 10GB
-            return 12;
+    /**
+     * Calculate smooth segments per tier using logarithmic interpolation
+     */
+    private double calculateSmoothSegmentsPerTier(long shardSizeBytes) {
+        double logSize = Math.log10(shardSizeBytes);
+
+        if (logSize < 8.0) { // < 100MB
+            return 5.0;
+        } else if (logSize < 9.0) { // 100MB - 1GB
+            double ratio = (logSize - 8.0) / 1.0;
+            return 5.0 + ratio * (8.0 - 5.0);
+        } else if (logSize < 10.0) { // 1GB - 10GB
+            double ratio = (logSize - 9.0) / 1.0;
+            return 8.0 + ratio * (10.0 - 8.0);
+        } else if (logSize < 11.0) { // 10GB - 100GB
+            double ratio = (logSize - 10.0) / 1.0;
+            return 10.0 + ratio * (12.0 - 10.0);
+        } else { // >= 100GB
+            return 12.0;
         }
     }