Read coursier env vars and Java properties via input tasks (#5793)

This is a new attempt at fixing
#5134. Turns out setting the
context class loader when evaluating user code addresses the issue while
not running into #5398 again.

---

Description from #5350 that
still applies:

This makes Mill read all environment variables and Java properties used
by coursier via a `Task.Input`.

In more detail, this adds a `CoursierConfigModule` external module in
scalalib. It contains a single `Task.Input`, that reads all environment
variables used by coursier via `Task.env` and all Java properties used
by coursier via `sys.props`.

Helper tasks compute default values for various parameters
(repositories, cache location, etc.).
`CoursierConfigModule.coursierConfig` returns most of these parameters
in a case class defined in Mill, `CoursierConfig`. For convenience,
down-the-line, we pass `CoursierConfig` instances around.

Fixes #5134 (by managing
coursier env vars / Java properties via `Task.Input`)

Author: alexarchambault

core/exec/src/mill/exec/GroupExecution.scala

@@ -320,7 +320,8 @@ private trait GroupExecution {
             counterMsg,
             destCreator,
             getEvaluator().asInstanceOf[Evaluator],
-            terminal
+            terminal,
+            rootModule.getClass.getClassLoader
           ) {
             try {
               task.evaluate(args) match {
@@ -533,7 +534,8 @@ private object GroupExecution {
       counterMsg: String,
       destCreator: DestCreator,
       evaluator: Evaluator,
-      terminal: Task[?]
+      terminal: Task[?],
+      classLoader: ClassLoader
   )(t: => T): T = {
     val isCommand = terminal.isInstanceOf[Task.Command[?]]
     val isInput = terminal.isInstanceOf[Task.Input[?]]
@@ -578,11 +580,18 @@ private object GroupExecution {
             )
 
           Evaluator.withCurrentEvaluator(exposedEvaluator) {
-            if (!exclusive) t
-            else {
-              logger.prompt.reportKey(Seq(counterMsg))
-              logger.prompt.withPromptPaused {
-                t
+            // Ensure the class loader used to load user code
+            // is set as context class loader when running user code.
+            // This is useful if users rely on libraries that look
+            // for resources added by other libraries, by using
+            // using java.util.ServiceLoader for example.
+            mill.api.ClassLoader.withContextClassLoader(classLoader) {
+              if (!exclusive) t
+              else {
+                logger.prompt.reportKey(Seq(counterMsg))
+                logger.prompt.withPromptPaused {
+                  t
+                }
               }
             }
           }

integration/bootstrap/no-java-bootstrap/src/NoJavaBootstrapTests.scala

@@ -19,7 +19,7 @@ object NoJavaBootstrapTests extends UtestIntegrationTestSuite {
     val cache = FileCache()
     val index = JvmIndex.load(
       cache = cache,
-      repositories = Resolve().repositories,
+      repositories = Resolve.defaultRepositories,
       indexChannel = JvmChannel.module(
         JvmChannel.centralModule(),
         version = mill.api.BuildInfo.coursierJvmIndexVersion

integration/feature/cs-env-vars/resources/build.mill

@@ -0,0 +1,13 @@
+package build
+import mill._, scalalib._
+import java.io.File
+
+object `package` extends ScalaModule {
+  def scalaVersion = "2.13.16"
+  def mvnDeps = Seq(
+    mvn"org.slf4j:slf4j-simple:2.0.9"
+  )
+  def printRunClasspath() = Task.Command {
+    println(runClasspath().map(_.path.toString).mkString(File.pathSeparator))
+  }
+}

integration/feature/cs-env-vars/src/CsEnvVarsTests.scala

@@ -0,0 +1,74 @@
+package mill.integration
+
+import coursier.cache.FileCache
+import mill.testkit.UtestIntegrationTestSuite
+import utest._
+
+import java.io.File
+
+object CsEnvVarsTests extends UtestIntegrationTestSuite {
+  val tests: Tests = Tests {
+    test("cache") - integrationTest { tester =>
+      import tester._
+
+      def check(cacheOpt: Option[os.Path]): Unit = {
+        val env = cacheOpt.toSeq.map { cache =>
+          "COURSIER_CACHE" -> cache.toString
+        }
+        val res = eval(
+          ("printRunClasspath"),
+          env = env.toMap
+        )
+        assert(res.exitCode == 0)
+
+        val cp = res.out.split(File.pathSeparator).filter(_.nonEmpty).map(os.Path(_))
+
+        val actualCache = cacheOpt.getOrElse(os.Path(FileCache().location))
+        assert(cp.exists(p => p.startsWith(actualCache) && p.last.startsWith("slf4j-api-")))
+      }
+
+      check(None)
+      check(Some(workspacePath / "cache-0"))
+      check(Some(workspacePath / "cache-1"))
+    }
+
+    test("mirrors") - integrationTest { tester =>
+      import tester._
+
+      def check(mirrorFileOpt: Option[os.Path]): Unit = {
+        val env = mirrorFileOpt.toSeq.map { file =>
+          "COURSIER_MIRRORS" -> file.toString
+        }
+        val res = eval(
+          ("printRunClasspath"),
+          env = env.toMap
+        )
+        assert(res.exitCode == 0)
+
+        val cacheRoot = os.Path(FileCache().location)
+        val cp = res.out.split(File.pathSeparator)
+          .filter(_.nonEmpty)
+          .map(os.Path(_))
+          .filter(_.startsWith(cacheRoot))
+          .map(_.relativeTo(cacheRoot).asSubPath)
+
+        val centralReplaced = cp.forall { f =>
+          f.startsWith(os.sub / "https/repo.maven.apache.org/maven2")
+        }
+        assert(cp.exists(f => f.last.startsWith("scala-library-") && f.last.endsWith(".jar")))
+        assert(mirrorFileOpt.nonEmpty == centralReplaced)
+      }
+
+      check(None)
+
+      val mirrorFile = workspacePath / "mirror.properties"
+      os.write(
+        mirrorFile,
+        """central.from=https://repo1.maven.org/maven2
+          |central.to=https://repo.maven.apache.org/maven2/
+          |""".stripMargin
+      )
+      check(Some(mirrorFile))
+    }
+  }
+}

integration/ide/bsp-server/resources/snapshots/logging

@@ -3,10 +3,10 @@
 [1-buildInitialize] Entered buildInitialize
 [1-buildInitialize] Got client semanticdbVersion: * Enabling SemanticDB support.
 [1-buildInitialize] buildInitialize took * msec
-[bsp-init-mill-build/build.mill-58] [info] compiling * Scala sources to * ...
-[bsp-init-mill-build/build.mill-58] [info] done compiling
-[bsp-init-build.mill-58] [info] compiling * Scala sources to * ...
-[bsp-init-build.mill-58] [info] done compiling
+[bsp-init-mill-build/build.mill-59] [info] compiling * Scala sources to * ...
+[bsp-init-mill-build/build.mill-59] [info] done compiling
+[bsp-init-build.mill-59] [info] compiling * Scala sources to * ...
+[bsp-init-build.mill-59] [info] done compiling
 [bsp-init] SNAPSHOT
 [2-workspaceBuildTargets] Entered workspaceBuildTargets
 [2-workspaceBuildTargets] Evaluating * tasks
@@ -31,12 +31,12 @@
 [4-compile] buildTargetCompile took * msec
 [5-compile] Entered buildTargetCompile
 [5-compile] Evaluating 1 task
-[5-compile-54] [info] compiling * Scala source to * ...
-[5-compile-54] [error] *:2:3: not found: value nope
-[5-compile-54] [error]   nope
-[5-compile-54] [error]   ^
-[5-compile-54] [error] one error found
-[5-compile-54] errored.compilation-error.semanticDbDataDetailed failed
+[5-compile-55] [info] compiling * Scala source to * ...
+[5-compile-55] [error] *:2:3: not found: value nope
+[5-compile-55] [error]   nope
+[5-compile-55] [error]   ^
+[5-compile-55] [error] one error found
+[5-compile-55] errored.compilation-error.semanticDbDataDetailed failed
 [5-compile] Done
 [5-compile] buildTargetCompile took * msec
 [6-compile] Entered buildTargetCompile

libs/init/gradle/src/mill/main/gradle/GradleBuildGenMain.scala

@@ -5,7 +5,7 @@ import mill.api.daemon.internal.internal
 import mill.main.buildgen.*
 import mill.main.buildgen.BuildGenUtil.*
 import mill.main.gradle.JavaModel.{Dep, ExternalDep}
-import mill.util.Jvm
+import mill.util.{CoursierConfig, Jvm}
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.tooling.GradleConnector
 import os.Path
@@ -59,7 +59,10 @@ object GradleBuildGenMain extends BuildGenBase.MavenAndGradle[ProjectModel, Dep]
     val args =
       cfg.shared.basicConfig.jvmId.map { id =>
         println(s"resolving Java home for jvmId $id")
-        val home = Jvm.resolveJavaHome(id).get
+        val home = Jvm.resolveJavaHome(
+          id,
+          config = CoursierConfig.default()
+        ).get
         s"-Dorg.gradle.java.home=$home"
       } ++ Seq("--init-script", writeGradleInitScript.toString())
 

libs/javalib/src/mill/javalib/CoursierConfigModule.scala

@@ -0,0 +1,180 @@
+package mill.javalib
+
+import coursier.CoursierEnv
+import coursier.cache.{CacheEnv, CachePolicy}
+import coursier.core.Repository
+import coursier.credentials.Credentials
+import coursier.params.Mirror
+import coursier.util.{EnvEntry, EnvValues}
+import mill.api.*
+import mill.{T, Task}
+
+import scala.concurrent.duration.Duration
+
+/**
+ * Reads coursier environment variables and Java properties
+ *
+ * This module reads environment variables and Java properties, and uses
+ * them to get default values for a number of coursier parameters, such as:
+ *
+ * * the coursier cache location, read from `COURSIER_CACHE` in the environment
+ *   and `coursier.cache` in Java properties. It is recommended to use an
+ *   absolute path rather than a relative one.
+ *
+ * * the coursier archive cache location, read from `COURSIER_ARCHIVE_CACHE` in
+ *   the environment and in `coursier.archive.cache` in Java properties.
+ *
+ * * credentials, read from `COURSIER_CREDENTIALS` in the environment, and
+ *   `coursier.credentials` in Java properties. Its format is documented
+ *   here: https://github.com/coursier/coursier/blob/701e93664855709db38e443cb7a19e36ddc49b78/doc/docs/other-credentials.md
+ *
+ * * TTL for snapshot artifacts / version listings / etc., read from
+ *   `COURSIER_TTL` in the environment and `coursier.ttl` in Java properties.
+ *   This value is parsed with `scala.concurrent.duration.Duration`, and accepts
+ *   formats like "5 hours", "1 day", etc. or just "0".
+ *
+ * * default repositories, read from `COURSIER_REPOSITORIES` in the environment
+ *   and `coursier.repositories` in Java properties. It can be set to values like
+ *   "ivy2Local|central|https://maven.google.com". Its format is documented here:
+ *   https://github.com/coursier/coursier/blob/701e93664855709db38e443cb7a19e36ddc49b78/doc/docs/other-repositories.md
+ *
+ * * mirror repository files, read from `COURSIER_MIRRORS` and `coursier.mirrors`,
+ *   whose file they point to should contains things like
+ *
+ *       google.from=https://repo1.maven.org/maven2
+ *       google.to=https://maven.google.com
+ *
+ * Environment variables always have precedence over Java properties.
+ */
+object CoursierConfigModule extends ExternalModule with CoursierConfigModule {
+  lazy val millDiscover = Discover[this.type]
+}
+
+trait CoursierConfigModule extends Module {
+
+  // All environment / Java properties "entries" that we're going to read.
+  // These are read all at once via coursierEnv.
+  private val entries = Seq(
+    CacheEnv.cache,
+    CacheEnv.credentials,
+    CacheEnv.ttl,
+    CacheEnv.cachePolicy,
+    CacheEnv.archiveCache,
+    CoursierEnv.mirrors,
+    CoursierEnv.mirrorsExtra,
+    CoursierEnv.repositories,
+    CoursierEnv.scalaCliConfig,
+    CoursierEnv.configDir
+  )
+
+  private val envVars = entries.map(_.envName).toSet
+  private val propNames = entries.map(_.propName).toSet
+
+  extension (entry: EnvEntry)
+    private def readFrom(env: Map[String, String], props: Map[String, String]): EnvValues =
+      EnvValues(env.get(entry.envName), props.get(entry.propName))
+
+  /**
+   * Environment variables and Java properties, and their values, used by coursier
+   */
+  def coursierEnv: T[(env: Map[String, String], props: Map[String, String])] = Task.Input {
+    val env = Task.env.filterKeys(envVars).toMap
+    val props =
+      propNames.iterator.flatMap(name => sys.props.get(name).iterator.map(name -> _)).toMap
+    (env, props)
+  }
+
+  /** Default repositories for dependency resolution */
+  def defaultRepositories: Task[Seq[Repository]] = Task.Anon {
+    val (env, props) = coursierEnv()
+    CoursierEnv.defaultRepositories(
+      CoursierEnv.repositories.readFrom(env, props),
+      CoursierEnv.scalaCliConfig.readFrom(env, props)
+    )
+  }
+
+  /** Default JSON configuration files to be used by coursier */
+  def defaultConfFiles: Task[Seq[PathRef]] = Task.Anon {
+    val (env, props) = coursierEnv()
+    CacheEnv.defaultConfFiles(CacheEnv.scalaCliConfig.readFrom(env, props))
+      .map(os.Path(_, BuildCtx.workspaceRoot))
+      .map(PathRef(_))
+  }
+
+  /** Default mirrors to be used by coursier */
+  def defaultMirrors: Task[Seq[Mirror]] = Task.Anon {
+    val (env, props) = coursierEnv()
+    CoursierEnv.defaultMirrors(
+      CoursierEnv.mirrors.readFrom(env, props),
+      CoursierEnv.mirrorsExtra.readFrom(env, props),
+      CoursierEnv.scalaCliConfig.readFrom(env, props),
+      CoursierEnv.configDir.readFrom(env, props)
+    )
+  }
+
+  /** Default cache location to be used by coursier */
+  def defaultCacheLocation: Task[String] = Task.Anon {
+    val (env, props) = coursierEnv()
+    val path: java.nio.file.Path = CacheEnv.defaultCacheLocation(
+      CacheEnv.cache.readFrom(env, props)
+    )
+    path.toString
+  }
+
+  /**
+   * Default archive cache location to be used by coursier
+   *
+   * This is the directory under which archives are unpacked
+   * by `coursier.cache.ArchiveCache`. This includes JVMs
+   * managed by coursier, which Mill relies on by default.
+   */
+  def defaultArchiveCacheLocation: Task[String] = Task.Anon {
+    val (env, props) = coursierEnv()
+    val path: java.nio.file.Path = CacheEnv.defaultArchiveCacheLocation(
+      CacheEnv.archiveCache.readFrom(env, props)
+    )
+    path.toString
+  }
+
+  /** Default repository credentials to be used by coursier */
+  def defaultCredentials: Task[Seq[Credentials]] = Task.Anon {
+    val (env, props) = coursierEnv()
+    CacheEnv.defaultCredentials(
+      CacheEnv.credentials.readFrom(env, props),
+      CacheEnv.scalaCliConfig.readFrom(env, props),
+      CacheEnv.configDir.readFrom(env, props)
+    )
+  }
+
+  /** Default TTL used by coursier when downloading changing artifacts such as snapshots */
+  def defaultTtl: Task[Option[Duration]] = Task.Anon {
+    val (env, props) = coursierEnv()
+    CacheEnv.defaultTtl(
+      CacheEnv.ttl.readFrom(env, props)
+    )
+  }
+
+  /** Default cache policies used by coursier when downloading artifacts */
+  def defaultCachePolicies: Task[Seq[CachePolicy]] = Task.Anon {
+    val (env, props) = coursierEnv()
+    CacheEnv.defaultCachePolicies(
+      CacheEnv.cachePolicy.readFrom(env, props)
+    )
+  }
+
+  /**
+   * Aggregates coursier parameters read from the environment
+   */
+  def coursierConfig: Task[mill.util.CoursierConfig] = Task.Anon {
+    mill.util.CoursierConfig(
+      defaultRepositories(),
+      defaultConfFiles().map(_.path),
+      defaultMirrors(),
+      defaultCacheLocation(),
+      defaultArchiveCacheLocation(),
+      defaultCredentials(),
+      defaultTtl(),
+      defaultCachePolicies()
+    )
+  }
+}