Limit/config concurrent downloads for CF and Modrinth (#610)

Author: itzg

src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java

@@ -133,6 +133,9 @@ public class CurseForgeInstaller {
     @Getter @Setter
     private ForgeUrlArgs forgeUrlArgs;
 
+    @Getter @Setter
+    int maxConcurrentDownloads;
+
     /**
      */
     public void installFromModpackZip(Path modpackZip, String slug) {
@@ -546,6 +549,8 @@ private ModPackResults processModpack(InstallContext context,
         final ExcludeIncludeIds excludeIncludeIds = resolveExcludeIncludes(context);
         log.debug("Using {}", excludeIncludeIds);
 
+        log.debug("Max concurrent downloads is {}", maxConcurrentDownloads);
+
         // Go through all the files listed in modpack (given project ID + file ID)
         final List<PathWithInfo> modFiles = Flux.fromIterable(modpackManifest.getFiles())
             // ...does the modpack even say it's required?
@@ -577,7 +582,8 @@ private ModPackResults processModpack(InstallContext context,
                 processFileWithIds(context, outputSubdirResolver,
                     excludeIncludeIds.getForceIncludeIds(), fileRef.getProjectID(), fileRef.getFileID()
                 )
-                    .checkpoint()
+                    .checkpoint(),
+                maxConcurrentDownloads > 0 ? maxConcurrentDownloads : 10
             )
             .collectList()
             .block();

src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java

@@ -171,6 +171,11 @@ static class Listed {
     @ArgGroup(exclusive = false)
     ForgeUrlArgs forgeUrlArgs = new ForgeUrlArgs();
 
+    @Option(names = {"--max-concurrent-downloads"}, defaultValue = "${env:CF_MAX_CONCURRENT_DOWNLOADS:-10}",
+        description = "Default is ${DEFAULT-VALUE}"
+    )
+    int maxConcurrentDownloads;
+
     @Override
     public Integer call() throws Exception {
         // https://www.curseforge.com/minecraft/modpacks/all-the-mods-8/files
@@ -208,7 +213,8 @@ public Integer call() throws Exception {
             .setDownloadsRepo(downloadsRepo)
             .setDisableApiCaching(disableApiCaching)
             .setCacheArgs(cacheArgs)
-            .setForgeUrlArgs(forgeUrlArgs);
+            .setForgeUrlArgs(forgeUrlArgs)
+            .setMaxConcurrentDownloads(maxConcurrentDownloads);
 
         if (apiBaseUrl != null) {
             installer.setApiBaseUrl(apiBaseUrl);

src/main/java/me/itzg/helpers/modrinth/InstallModrinthModpackCommand.java

@@ -121,6 +121,12 @@ public class InstallModrinthModpackCommand implements Callable<Integer> {
     @ArgGroup(exclusive = false)
     ForgeUrlArgs forgeUrlArgs = new ForgeUrlArgs();
 
+    @Option(names = {"--max-concurrent-downloads"}, defaultValue = "${env:MODRINTH_MAX_CONCURRENT_DOWNLOADS:-10}",
+        description = "Can also set env var MODRINTH_MAX_CONCURRENT_DOWNLOADS%n"
+            + "Default is ${DEFAULT-VALUE}"
+    )
+    int maxConcurrentDownloads = 1;
+
     @Override
     public Integer call() throws IOException {
 
@@ -154,6 +160,7 @@ public Integer call() throws IOException {
                             )
                         )
                         .setOverridesExclusions(overridesExclusions)
+                        .setMaxConcurrentDownloads(maxConcurrentDownloads)
                         .processModpack(sharedFetch)
                         .flatMap(installation -> {
                             if (resultsFile != null) {

src/main/java/me/itzg/helpers/modrinth/ModrinthPackInstaller.java

@@ -13,6 +13,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.zip.ZipFile;
+import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import me.itzg.helpers.errors.GenericException;
@@ -46,8 +47,10 @@ public class ModrinthPackInstaller {
     private final boolean forceModloaderReinstall;
     private final FileInclusionCalculator fileInclusionCalculator;
     private final Options sharedFetchOpts;
-    @Setter
+    @Setter @Getter
     private ForgeUrlArgs forgeUrlArgs = new ForgeUrlArgs();
+    @Setter @Getter
+    private int maxConcurrentDownloads = 1;
 
     private AntPathMatcher overridesExclusions;
 
@@ -169,7 +172,7 @@ private Flux<Path> processModFiles(ModpackIndex modpackIndex) {
                     (uri, file, contentSizeBytes) ->
                         log.info("Downloaded {}", modpackFilePath)
                 );
-            });
+            }, maxConcurrentDownloads);
     }
 
     @SuppressWarnings("SameParameterValue")