Skip to content

Commit 97de675

Browse files
mayanksharma27mayanksharma27
committed
Added Changes to support pinning in FileCache-RefCountedCache
Signed-off-by: Mayank Sharma <[email protected]>
1 parent aec3fe9 commit 97de675

20 files changed

+568
-133
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3030
- Add support for `matched_fields` with the unified highlighter ([#18164](https://github.com/opensearch-project/OpenSearch/issues/18164))
3131
- [repository-s3] Add support for SSE-KMS and S3 bucket owner verification ([#18312](https://github.com/opensearch-project/OpenSearch/pull/18312))
3232
- Added File Cache Stats - Involves Block level as well as full file level stats ([#17538](https://github.com/opensearch-project/OpenSearch/issues/17479))
33+
- Added File Cache Pinning ([#17617](https://github.com/opensearch-project/OpenSearch/issues/13648))
3334

3435
### Changed
3536
- Create generic DocRequest to better categorize ActionRequests ([#18269](https://github.com/opensearch-project/OpenSearch/pull/18269)))
36-
37+
3738
### Dependencies
3839
- Update Apache Lucene from 10.1.0 to 10.2.1 ([#17961](https://github.com/opensearch-project/OpenSearch/pull/17961))
3940
- Bump `com.google.code.gson:gson` from 2.12.1 to 2.13.1 ([#17923](https://github.com/opensearch-project/OpenSearch/pull/17923), [#18266](https://github.com/opensearch-project/OpenSearch/pull/18266))

‎server/src/main/java/org/opensearch/index/store/CompositeDirectory.java‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -327,19 +327,15 @@ public String toString() {
327327
*/
328328
public void afterSyncToRemote(String file) {
329329
ensureOpen();
330-
/*
331-
Decrementing the refCount here for the path so that it becomes eligible for eviction
332-
This is a temporary solution until pinning support is added
333-
TODO - Unpin the files here from FileCache so that they become eligible for eviction, once pinning/unpinning support is added in FileCache
334-
Uncomment the below commented line(to remove the file from cache once uploaded) to test block based functionality
335-
*/
330+
336331
logger.trace(
337332
"Composite Directory[{}]: File {} uploaded to Remote Store and now can be eligible for eviction in FileCache",
338333
this::toString,
339334
() -> file
340335
);
341-
fileCache.decRef(getFilePath(file));
342-
// fileCache.remove(getFilePath(fileName));
336+
final Path filePath = getFilePath(file);
337+
fileCache.unpin(filePath);
338+
// fileCache.remove(filePath);
343339
}
344340

345341
// Visibility public since we need it in IT tests
@@ -390,12 +386,10 @@ private String[] getRemoteFiles() throws IOException {
390386

391387
protected void cacheFile(String name) throws IOException {
392388
Path filePath = getFilePath(name);
393-
// put will increase the refCount for the path, making sure it is not evicted, will decrease the ref after it is uploaded to Remote
394-
// so that it can be evicted after that
395-
// this is just a temporary solution, will pin the file once support for that is added in FileCache
396-
// TODO : Pin the above filePath in the file cache once pinning support is added so that it cannot be evicted unless it has been
397-
// successfully uploaded to Remote
389+
398390
fileCache.put(filePath, new CachedFullFileIndexInput(fileCache, filePath, localDirectory.openInput(name, IOContext.DEFAULT)));
391+
fileCache.pin(filePath);
392+
fileCache.decRef(filePath);
399393
}
400394

401395
}

‎server/src/main/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStats.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,24 +39,28 @@ public class AggregateFileCacheStats implements Writeable, ToXContentFragment {
3939
private final FileCacheStats overallFileCacheStats;
4040
private final FileCacheStats fullFileCacheStats;
4141
private final FileCacheStats blockFileCacheStats;
42+
private final FileCacheStats pinnedFileCacheStats;
4243

4344
public AggregateFileCacheStats(
4445
final long timestamp,
4546
final FileCacheStats overallFileCacheStats,
4647
final FileCacheStats fullFileCacheStats,
47-
final FileCacheStats blockFileCacheStats
48+
final FileCacheStats blockFileCacheStats,
49+
FileCacheStats pinnedFileCacheStats
4850
) {
4951
this.timestamp = timestamp;
5052
this.overallFileCacheStats = overallFileCacheStats;
5153
this.fullFileCacheStats = fullFileCacheStats;
5254
this.blockFileCacheStats = blockFileCacheStats;
55+
this.pinnedFileCacheStats = pinnedFileCacheStats;
5356
}
5457

5558
public AggregateFileCacheStats(final StreamInput in) throws IOException {
5659
this.timestamp = in.readLong();
5760
this.overallFileCacheStats = new FileCacheStats(in);
5861
this.fullFileCacheStats = new FileCacheStats(in);
5962
this.blockFileCacheStats = new FileCacheStats(in);
63+
this.pinnedFileCacheStats = new FileCacheStats(in);
6064
}
6165

6266
public static short calculatePercentage(long used, long max) {
@@ -69,6 +73,7 @@ public void writeTo(final StreamOutput out) throws IOException {
6973
overallFileCacheStats.writeTo(out);
7074
fullFileCacheStats.writeTo(out);
7175
blockFileCacheStats.writeTo(out);
76+
pinnedFileCacheStats.writeTo(out);
7277
}
7378

7479
public long getTimestamp() {
@@ -91,6 +96,10 @@ public ByteSizeValue getUsed() {
9196
return new ByteSizeValue(overallFileCacheStats.getUsed());
9297
}
9398

99+
public ByteSizeValue getPinnedUsage() {
100+
return new ByteSizeValue(overallFileCacheStats.getPinnedUsage());
101+
}
102+
94103
public short getUsedPercent() {
95104
return calculatePercentage(getUsed().getBytes(), getTotal().getBytes());
96105
}
@@ -114,6 +123,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
114123
builder.humanReadableField(Fields.ACTIVE_IN_BYTES, Fields.ACTIVE, getActive());
115124
builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal());
116125
builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, getUsed());
126+
builder.humanReadableField(Fields.PINNED_IN_BYTES, Fields.PINNED, getPinnedUsage());
117127
builder.humanReadableField(Fields.EVICTIONS_IN_BYTES, Fields.EVICTIONS, getEvicted());
118128
builder.field(Fields.ACTIVE_PERCENT, getActivePercent());
119129
builder.field(Fields.USED_PERCENT, getUsedPercent());
@@ -122,7 +132,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
122132
overallFileCacheStats.toXContent(builder, params);
123133
fullFileCacheStats.toXContent(builder, params);
124134
blockFileCacheStats.toXContent(builder, params);
125-
135+
pinnedFileCacheStats.toXContent(builder, params);
126136
builder.endObject();
127137
return builder;
128138
}
@@ -134,6 +144,8 @@ static final class Fields {
134144
static final String ACTIVE_IN_BYTES = "active_in_bytes";
135145
static final String USED = "used";
136146
static final String USED_IN_BYTES = "used_in_bytes";
147+
static final String PINNED = "pinned";
148+
static final String PINNED_IN_BYTES = "pinned_in_bytes";
137149
static final String EVICTIONS = "evictions";
138150
static final String EVICTIONS_IN_BYTES = "evictions_in_bytes";
139151
static final String TOTAL = "total";
@@ -153,7 +165,8 @@ static final class Fields {
153165
public enum FileCacheStatsType {
154166
FULL_FILE_STATS("full_file_stats"),
155167
BLOCK_FILE_STATS("block_file_stats"),
156-
OVER_ALL_STATS("over_all_stats");
168+
OVER_ALL_STATS("over_all_stats"),
169+
PINNED_FILE_STATS("pinned_file_stats");
157170

158171
private final String fileCacheStatsType;
159172

‎server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,26 @@ public void decRef(Path key) {
124124
theCache.decRef(key);
125125
}
126126

127+
/**
128+
* Pins the key in the cache, preventing it from being evicted.
129+
*
130+
* @param key
131+
*/
132+
@Override
133+
public void pin(Path key) {
134+
theCache.pin(key);
135+
}
136+
137+
/**
138+
* Unpins the key in the cache, allowing it to be evicted.
139+
*
140+
* @param key
141+
*/
142+
@Override
143+
public void unpin(Path key) {
144+
theCache.unpin(key);
145+
}
146+
127147
@Override
128148
public Integer getRef(Path key) {
129149
return theCache.getRef(key);
@@ -141,14 +161,22 @@ public long prune(Predicate<Path> keyPredicate) {
141161

142162
@Override
143163
public long usage() {
144-
long l = theCache.usage();
145-
return l;
164+
return theCache.usage();
146165
}
147166

148167
@Override
149168
public long activeUsage() {
150-
long l = theCache.activeUsage();
151-
return l;
169+
return theCache.activeUsage();
170+
}
171+
172+
/**
173+
* Returns the pinned usage of this cache.
174+
*
175+
* @return the combined pinned weight of the values in this cache.
176+
*/
177+
@Override
178+
public long pinnedUsage() {
179+
return theCache.pinnedUsage();
152180
}
153181

154182
@Override
@@ -225,12 +253,14 @@ public AggregateFileCacheStats fileCacheStats() {
225253
final RefCountedCacheStats overallCacheStats = stats.getOverallCacheStats();
226254
final RefCountedCacheStats fullFileCacheStats = stats.getFullFileCacheStats();
227255
final RefCountedCacheStats blockFileCacheStats = stats.getBlockFileCacheStats();
256+
final RefCountedCacheStats pinnedFileCacheStats = stats.getPinnedFileCacheStats();
228257
return new AggregateFileCacheStats(
229258
System.currentTimeMillis(),
230259
new FileCacheStats(
231260
overallCacheStats.activeUsage(),
232261
capacity(),
233262
overallCacheStats.usage(),
263+
overallCacheStats.pinnedUsage(),
234264
overallCacheStats.evictionWeight(),
235265
overallCacheStats.hitCount(),
236266
overallCacheStats.missCount(),
@@ -240,6 +270,7 @@ public AggregateFileCacheStats fileCacheStats() {
240270
fullFileCacheStats.activeUsage(),
241271
capacity(),
242272
fullFileCacheStats.usage(),
273+
fullFileCacheStats.pinnedUsage(),
243274
fullFileCacheStats.evictionWeight(),
244275
fullFileCacheStats.hitCount(),
245276
fullFileCacheStats.missCount(),
@@ -249,10 +280,21 @@ public AggregateFileCacheStats fileCacheStats() {
249280
blockFileCacheStats.activeUsage(),
250281
capacity(),
251282
blockFileCacheStats.usage(),
283+
blockFileCacheStats.pinnedUsage(),
252284
blockFileCacheStats.evictionWeight(),
253285
blockFileCacheStats.hitCount(),
254286
blockFileCacheStats.missCount(),
255287
FileCacheStatsType.BLOCK_FILE_STATS
288+
),
289+
new FileCacheStats(
290+
pinnedFileCacheStats.activeUsage(),
291+
capacity(),
292+
pinnedFileCacheStats.usage(),
293+
pinnedFileCacheStats.pinnedUsage(),
294+
pinnedFileCacheStats.evictionWeight(),
295+
pinnedFileCacheStats.hitCount(),
296+
pinnedFileCacheStats.missCount(),
297+
FileCacheStatsType.PINNED_FILE_STATS
256298
)
257299
);
258300
}

‎server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public class FileCacheStats implements Writeable, ToXContentFragment {
3838
private final long active;
3939
private final long total;
4040
private final long used;
41+
private final long pinned;
4142
private final long evicted;
4243
private final long hits;
4344
private final long misses;
@@ -47,6 +48,7 @@ public FileCacheStats(
4748
final long active,
4849
long total,
4950
final long used,
51+
final long pinned,
5052
final long evicted,
5153
final long hits,
5254
long misses,
@@ -55,6 +57,7 @@ public FileCacheStats(
5557
this.active = active;
5658
this.total = total;
5759
this.used = used;
60+
this.pinned = pinned;
5861
this.evicted = evicted;
5962
this.hits = hits;
6063
this.misses = misses;
@@ -66,6 +69,7 @@ public FileCacheStats(final StreamInput in) throws IOException {
6669
this.active = in.readLong();
6770
this.total = in.readLong();
6871
this.used = in.readLong();
72+
this.pinned = in.readLong();
6973
this.evicted = in.readLong();
7074
this.hits = in.readLong();
7175
this.misses = in.readLong();
@@ -77,6 +81,7 @@ public void writeTo(final StreamOutput out) throws IOException {
7781
out.writeLong(active);
7882
out.writeLong(total);
7983
out.writeLong(used);
84+
out.writeLong(pinned);
8085
out.writeLong(evicted);
8186
out.writeLong(hits);
8287
out.writeLong(misses);
@@ -106,6 +111,10 @@ public long getTotal() {
106111
return total;
107112
}
108113

114+
public long getPinnedUsage() {
115+
return pinned;
116+
}
117+
109118
public short getUsedPercent() {
110119
return calculatePercentage(getUsed(), total);
111120
}
@@ -122,6 +131,7 @@ static final class Fields {
122131
static final String ACTIVE = "active";
123132
static final String ACTIVE_IN_BYTES = "active_in_bytes";
124133
static final String USED = "used";
134+
static final String PINNED = "pinned";
125135
static final String USED_IN_BYTES = "used_in_bytes";
126136
static final String EVICTIONS = "evictions";
127137
static final String EVICTIONS_IN_BYTES = "evictions_in_bytes";
@@ -134,6 +144,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
134144
builder.startObject(statsType.toString());
135145
builder.humanReadableField(FileCacheStats.Fields.ACTIVE_IN_BYTES, FileCacheStats.Fields.ACTIVE, new ByteSizeValue(getActive()));
136146
builder.humanReadableField(FileCacheStats.Fields.USED_IN_BYTES, FileCacheStats.Fields.USED, new ByteSizeValue(getUsed()));
147+
builder.humanReadableField(FileCacheStats.Fields.USED_IN_BYTES, Fields.PINNED, new ByteSizeValue(getPinnedUsage()));
137148
builder.humanReadableField(
138149
FileCacheStats.Fields.EVICTIONS_IN_BYTES,
139150
FileCacheStats.Fields.EVICTIONS,

0 commit comments

Comments
 (0)