Skip to content

Commit 0a75343

Browse files
itzgitzg
authored
curseforge: cache get mod info and file API calls to disk (#480)
1 parent de7dc99 commit 0a75343

File tree

13 files changed

+2587
-47
lines changed

13 files changed

+2587
-47
lines changed

‎src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java‎

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import java.util.Collection;
1010
import java.util.HashMap;
1111
import java.util.Map;
12-
import java.util.concurrent.ConcurrentHashMap;
1312
import lombok.extern.slf4j.Slf4j;
1413
import me.itzg.helpers.curseforge.model.Category;
1514
import me.itzg.helpers.curseforge.model.CurseForgeFile;
@@ -22,6 +21,7 @@
2221
import me.itzg.helpers.curseforge.model.ModsSearchResponse;
2322
import me.itzg.helpers.errors.GenericException;
2423
import me.itzg.helpers.errors.InvalidParameterException;
24+
import me.itzg.helpers.files.ApiCaching;
2525
import me.itzg.helpers.http.FailedRequestException;
2626
import me.itzg.helpers.http.Fetch;
2727
import me.itzg.helpers.http.FileDownloadStatusHandler;
@@ -35,6 +35,8 @@
3535
@Slf4j
3636
public class CurseForgeApiClient implements AutoCloseable {
3737

38+
public static final String CACHING_NAMESPACE = "curseforge";
39+
3840
public static final String CATEGORY_MODPACKS = "modpacks";
3941
public static final String CATEGORY_MC_MODS = "mc-mods";
4042
public static final String CATEGORY_BUKKIT_PLUGINS = "bukkit-plugins";
@@ -48,10 +50,12 @@ public class CurseForgeApiClient implements AutoCloseable {
4850
private final UriBuilder downloadFallbackUriBuilder;
4951
private final String gameId;
5052

51-
private final ConcurrentHashMap<Integer, CurseForgeMod> cachedMods = new ConcurrentHashMap<>();
53+
private final ApiCaching apiCaching;
5254

53-
public CurseForgeApiClient(String apiBaseUrl, String apiKey, SharedFetch.Options sharedFetchOptions, String gameId
55+
public CurseForgeApiClient(String apiBaseUrl, String apiKey, SharedFetch.Options sharedFetchOptions, String gameId,
56+
ApiCaching apiCaching
5457
) {
58+
this.apiCaching = apiCaching;
5559
if (apiKey == null || apiKey.trim().isEmpty()) {
5660
throw new InvalidParameterException("CurseForge API key is required");
5761
}
@@ -128,8 +132,7 @@ else if (searchResponse.getData().size() > 1) {
128132
else {
129133
return Mono.just(searchResponse.getData().get(0));
130134
}
131-
})
132-
.doOnNext(curseForgeMod -> cachedMods.put(curseForgeMod.getId(), curseForgeMod));
135+
});
133136
}
134137

135138
/**
@@ -184,42 +187,42 @@ public Mono<CurseForgeMod> getModInfo(
184187
) {
185188
log.debug("Getting mod metadata for {}", projectID);
186189

187-
final CurseForgeMod cached = cachedMods.get(projectID);
188-
if (cached != null) {
189-
return Mono.just(cached);
190-
}
191-
192-
return preparedFetch.fetch(
193-
uriBuilder.resolve("/v1/mods/{modId}", projectID)
194-
)
195-
.toObject(GetModResponse.class)
196-
.assemble()
197-
.checkpoint("Getting mod info for " + projectID)
198-
.map(GetModResponse::getData)
199-
.doOnNext(curseForgeMod -> cachedMods.put(curseForgeMod.getId(), curseForgeMod));
190+
return apiCaching.cache("getModInfo", CurseForgeMod.class,
191+
preparedFetch.fetch(
192+
uriBuilder.resolve("/v1/mods/{modId}", projectID)
193+
)
194+
.toObject(GetModResponse.class)
195+
.assemble()
196+
.checkpoint("Getting mod info for " + projectID)
197+
.map(GetModResponse::getData),
198+
projectID
199+
);
200200
}
201201

202202
public Mono<CurseForgeFile> getModFileInfo(
203203
int projectID, int fileID
204204
) {
205205
log.debug("Getting mod file metadata for {}:{}", projectID, fileID);
206206

207-
return preparedFetch.fetch(
208-
uriBuilder.resolve("/v1/mods/{modId}/files/{fileId}", projectID, fileID)
209-
)
210-
.toObject(GetModFileResponse.class)
211-
.assemble()
212-
.onErrorMap(FailedRequestException.class::isInstance, e -> {
213-
final FailedRequestException fre = (FailedRequestException) e;
214-
if (fre.getStatusCode() == 400) {
215-
if (isNotFoundResponse(fre.getBody())) {
216-
return new InvalidParameterException("Requested file not found for modpack", e);
207+
return apiCaching.cache("getModFileInfo", CurseForgeFile.class,
208+
preparedFetch.fetch(
209+
uriBuilder.resolve("/v1/mods/{modId}/files/{fileId}", projectID, fileID)
210+
)
211+
.toObject(GetModFileResponse.class)
212+
.assemble()
213+
.onErrorMap(FailedRequestException.class::isInstance, e -> {
214+
final FailedRequestException fre = (FailedRequestException) e;
215+
if (fre.getStatusCode() == 400) {
216+
if (isNotFoundResponse(fre.getBody())) {
217+
return new InvalidParameterException("Requested file not found for modpack", e);
218+
}
217219
}
218-
}
219-
return e;
220-
})
221-
.map(GetModFileResponse::getData)
222-
.checkpoint();
220+
return e;
221+
})
222+
.map(GetModFileResponse::getData)
223+
.checkpoint(),
224+
projectID, fileID
225+
);
223226
}
224227

225228
public Mono<Path> download(CurseForgeFile cfFile, Path outputFile, FileDownloadStatusHandler handler) {

‎src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
import me.itzg.helpers.curseforge.model.ModLoaderType;
2525
import me.itzg.helpers.errors.GenericException;
2626
import me.itzg.helpers.errors.InvalidParameterException;
27+
import me.itzg.helpers.files.ApiCaching;
28+
import me.itzg.helpers.files.ApiCachingImpl;
29+
import me.itzg.helpers.files.DisabledApiCaching;
2730
import me.itzg.helpers.files.Manifests;
2831
import me.itzg.helpers.http.SharedFetchArgs;
2932
import org.jetbrains.annotations.NotNull;
@@ -93,6 +96,9 @@ public void setSlugCategory(String defaultCategory) {
9396
)
9497
ModLoaderType modLoaderType;
9598

99+
@Option(names = "--disable-api-caching", defaultValue = "${env:CF_DISABLE_API_CACHING:-false}")
100+
boolean disableApiCaching;
101+
96102
@ArgGroup(exclusive = false)
97103
SharedFetchArgs sharedFetchArgs = new SharedFetchArgs();
98104

@@ -116,10 +122,14 @@ public Integer call() throws Exception {
116122
final CurseForgeFilesManifest newManifest;
117123

118124
if (modFileRefs != null && !modFileRefs.isEmpty()) {
119-
try (CurseForgeApiClient apiClient = new CurseForgeApiClient(
120-
apiBaseUrl, apiKey, sharedFetchArgs.options(),
121-
CurseForgeApiClient.MINECRAFT_GAME_ID
122-
)) {
125+
try (
126+
final ApiCaching apiCaching = disableApiCaching ? new DisabledApiCaching() : new ApiCachingImpl(outputDir, CACHING_NAMESPACE);
127+
final CurseForgeApiClient apiClient = new CurseForgeApiClient(
128+
apiBaseUrl, apiKey, sharedFetchArgs.options(),
129+
CurseForgeApiClient.MINECRAFT_GAME_ID,
130+
apiCaching
131+
)
132+
) {
123133
newManifest = apiClient.loadCategoryInfo(Arrays.asList(CATEGORY_MC_MODS, CATEGORY_BUKKIT_PLUGINS))
124134
.flatMap(categoryInfo ->
125135
processModFileRefs(categoryInfo, previousFiles, apiClient)

‎src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import static java.util.Collections.emptySet;
44
import static java.util.Objects.requireNonNull;
55
import static java.util.Optional.ofNullable;
6+
import static me.itzg.helpers.curseforge.CurseForgeApiClient.CACHING_NAMESPACE;
67
import static me.itzg.helpers.curseforge.CurseForgeApiClient.modFileDownloadStatusHandler;
78
import static me.itzg.helpers.singles.MoreCollections.safeStreamFrom;
89

@@ -44,6 +45,9 @@
4445
import me.itzg.helpers.errors.InvalidParameterException;
4546
import me.itzg.helpers.errors.RateLimitException;
4647
import me.itzg.helpers.fabric.FabricLauncherInstaller;
48+
import me.itzg.helpers.files.ApiCaching;
49+
import me.itzg.helpers.files.ApiCachingImpl;
50+
import me.itzg.helpers.files.DisabledApiCaching;
4751
import me.itzg.helpers.files.Manifests;
4852
import me.itzg.helpers.files.ResultsFileWriter;
4953
import me.itzg.helpers.forge.ForgeInstaller;
@@ -127,10 +131,12 @@ public class CurseForgeInstaller {
127131
@Getter @Setter
128132
private List<String> ignoreMissingFiles;
129133

134+
@Getter @Setter
135+
private boolean disableApiCaching;
136+
130137
/**
131-
* @throws MissingModsException if any mods need to be manually downloaded
132138
*/
133-
public void installFromModpackZip(Path modpackZip, String slug) throws IOException {
139+
public void installFromModpackZip(Path modpackZip, String slug) {
134140
requireNonNull(modpackZip, "modpackZip is required");
135141

136142
install(slug, context -> {
@@ -147,9 +153,8 @@ public void installFromModpackZip(Path modpackZip, String slug) throws IOExcepti
147153
}
148154

149155
/**
150-
* @throws MissingModsException if any mods need to be manually downloaded
151156
*/
152-
public void installFromModpackManifest(String modpackManifestLoc, String slug) throws IOException {
157+
public void installFromModpackManifest(String modpackManifestLoc, String slug) {
153158
requireNonNull(modpackManifestLoc, "modpackManifest is required");
154159

155160
install(slug, context -> {
@@ -184,7 +189,7 @@ public void install(String slug, String fileMatcher, Integer fileId) throws IOEx
184189
);
185190
}
186191

187-
void install(String slug, InstallationEntryPoint entryPoint) throws IOException {
192+
void install(String slug, InstallationEntryPoint entryPoint) {
188193
requireNonNull(outputDir, "outputDir is required");
189194
requireNonNull(slug);
190195
requireNonNull(entryPoint);
@@ -209,9 +214,11 @@ void install(String slug, InstallationEntryPoint entryPoint) throws IOException
209214
}
210215

211216
try (
212-
CurseForgeApiClient cfApi = new CurseForgeApiClient(
217+
final ApiCaching apiCaching = disableApiCaching ? new DisabledApiCaching() : new ApiCachingImpl(outputDir, CACHING_NAMESPACE);
218+
final CurseForgeApiClient cfApi = new CurseForgeApiClient(
213219
apiBaseUrl, apiKey, sharedFetchOptions,
214-
CurseForgeApiClient.MINECRAFT_GAME_ID
220+
CurseForgeApiClient.MINECRAFT_GAME_ID,
221+
apiCaching
215222
)
216223
) {
217224
final CategoryInfo categoryInfo = cfApi.loadCategoryInfo(applicableClassIdSlugs)
@@ -238,6 +245,8 @@ void install(String slug, InstallationEntryPoint entryPoint) throws IOException
238245
else {
239246
throw e;
240247
}
248+
} catch (IOException e) {
249+
throw new GenericException("Failed to setup API caching", e);
241250
}
242251
}
243252

‎src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,9 @@ static class Listed {
160160
@Option(names = "--missing-mods-filename", defaultValue = "MODS_NEED_DOWNLOAD.txt")
161161
String missingModsFilename;
162162

163+
@Option(names = "--disable-api-caching", defaultValue = "${env:CF_DISABLE_API_CACHING:-false}")
164+
boolean disableApiCaching;
165+
163166
@Override
164167
public Integer call() throws Exception {
165168
// https://www.curseforge.com/minecraft/modpacks/all-the-mods-8/files
@@ -194,7 +197,8 @@ public Integer call() throws Exception {
194197
.setOverridesExclusions(overridesExclusions)
195198
.setSharedFetchOptions(sharedFetchArgs.options())
196199
.setApiKey(apiKey)
197-
.setDownloadsRepo(downloadsRepo);
200+
.setDownloadsRepo(downloadsRepo)
201+
.setDisableApiCaching(disableApiCaching);
198202

199203
if (apiBaseUrl != null) {
200204
installer.setApiBaseUrl(apiBaseUrl);

‎src/main/java/me/itzg/helpers/files/ApiCaching.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package me.itzg.helpers.files;
2+
3+
import java.io.IOException;
4+
import reactor.core.publisher.Mono;
5+
6+
public interface ApiCaching extends AutoCloseable {
7+
8+
<R> Mono<R> cache(String operation, Class<R> returnType, Mono<R> resolver, Object... keys);
9+
10+
void close() throws IOException;
11+
}

0 commit comments

Comments
 (0)