Tracking implicit compiler dependencies

WORKSPACE

@@ -22,7 +22,7 @@ http_archive(
 
 load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config")
 
-scala_config()
+scala_config(enable_compiler_dependency_tracking = True)
 
 load("//scala:scala.bzl", "rules_scala_setup", "rules_scala_toolchain_deps_repositories")
 

dt_patches/dt_compiler_2.11.patch

@@ -0,0 +1,20 @@
+Subject: [PATCH] for patch
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git scala/tools/nsc/symtab/SymbolLoaders.scala scala/tools/nsc/symtab/SymbolLoaders.scala
+--- scala/tools/nsc/symtab/SymbolLoaders.scala	(revision 2e2f65a201b0b06e01a39fec4ae232c5b94efcca)
++++ scala/tools/nsc/symtab/SymbolLoaders.scala	(revision 21ecced66f88cff2a81a7f69340be63dd0bc9b88)
+@@ -210,6 +210,11 @@
+         val currentphase = phase
+         doComplete(root)
+         phase = currentphase
++        val assocFile = root.associatedFile
++        if (assocFile.underlyingSource.exists(_.isInstanceOf[scala.reflect.io.FileZipArchive])) {
++          val archivePath = assocFile.underlyingSource.map(_.path).headOption.getOrElse("N/A")
++          inform(s"DT:$archivePath")
++        }
+         informTime("loaded " + description, start)
+         ok = true
+         setSource(root)

dt_patches/dt_compiler_2.12.1.patch

@@ -0,0 +1,20 @@
+Subject: [PATCH] for patch
+---
+Index: scala/tools/nsc/symtab/SymbolLoaders.scala
+<+>UTF-8
+===================================================================
+diff --git scala/tools/nsc/symtab/SymbolLoaders.scala scala/tools/nsc/symtab/SymbolLoaders.scala
+--- scala/tools/nsc/symtab/SymbolLoaders.scala	(revision 8684ae833dcfeac6107343fcca5501301e509eef)
++++ scala/tools/nsc/symtab/SymbolLoaders.scala	(revision 74239a150e641ebfe220a6698dbc88c8c3f6f092)
+@@ -208,6 +208,11 @@
+         val currentphase = phase
+         doComplete(root)
+         phase = currentphase
++        val assocFile = root.associatedFile
++        if (assocFile.underlyingSource.exists(_.isInstanceOf[scala.reflect.io.FileZipArchive])) {
++          val archivePath = assocFile.underlyingSource.map(_.path).headOption.getOrElse("N/A")
++          inform(s"DT:$archivePath")
++        }
+         informTime("loaded " + description, start)
+         ok = true
+         setSource(root)

dt_patches/dt_compiler_2.12.8.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] for patch
+---
+Index: scala/tools/nsc/symtab/SymbolLoaders.scala
+<+>UTF-8
+===================================================================
+diff --git scala/tools/nsc/symtab/SymbolLoaders.scala scala/tools/nsc/symtab/SymbolLoaders.scala
+--- scala/tools/nsc/symtab/SymbolLoaders.scala	(revision 38cd84d976880eab16af17a44d186bbddcd554f2)
++++ scala/tools/nsc/symtab/SymbolLoaders.scala	(revision 58793f164e13a93ab9114cc30c61a24c12a2277f)
+@@ -230,6 +230,10 @@
+           val currentphase = phase
+           doComplete(root)
+           phase = currentphase
++          if (assocFile.underlyingSource.exists(_.isInstanceOf[scala.reflect.io.FileZipArchive])) {
++            val archivePath = assocFile.underlyingSource.map(_.path).headOption.getOrElse("N/A")
++            inform(s"DT:$archivePath")
++          }
+           informTime("loaded " + description, start)
+           ok = true
+           setSource(root)

dt_patches/dt_compiler_2.12.patch

@@ -0,0 +1,19 @@
+Index: src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala b/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala
+--- scala/tools/nsc/symtab/SymbolLoaders.scala	(revision 45183fa6a98f33ce468866a47d50e5508ac28ba5)
++++ scala/tools/nsc/symtab/SymbolLoaders.scala	(revision be2326549ffd607c3e57bb245e4c44d124438f21)
+@@ -249,6 +249,10 @@
+           val currentphase = phase
+           doComplete(root)
+           phase = currentphase
++          if (assocFile.underlyingSource.exists(_.isInstanceOf[scala.reflect.io.FileZipArchive])) {
++            val archivePath = assocFile.underlyingSource.map(_.path).headOption.getOrElse("N/A")
++            inform(s"DT:$archivePath")
++          }
+           if (settings.verbose) informTime("loaded " + description, start)
+           ok = true
+           setSource(root)

dt_patches/dt_compiler_2.13.patch

@@ -0,0 +1,21 @@
+Subject: [PATCH] for patch
+---
+Index: src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git scala/tools/nsc/symtab/SymbolLoaders.scala scala/tools/nsc/symtab/SymbolLoaders.scala
+--- scala/tools/nsc/symtab/SymbolLoaders.scala	(revision d578a02ea6b41b662072759c82c19f9309a15176)
++++ scala/tools/nsc/symtab/SymbolLoaders.scala	(revision 03634634be5aed4ec5053196bfdef102ad349f14)
+@@ -244,6 +244,10 @@
+       currentRunProfilerBeforeCompletion(root, assocFile)
+       try {
+         try {
++          if (assocFile.underlyingSource.exists(_.isInstanceOf[scala.reflect.io.FileZipArchive])) {
++            val archivePath = assocFile.underlyingSource.map(_.path).headOption.getOrElse("N/A")
++            inform(s"DT:$archivePath")
++          }
+           informingProgress("loaded " + description) {
+             val currentphase = phase
+             try doComplete(root)

dt_patches/dt_patch_test.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+NC='\033[0m'
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+
+run_test_local() {
+  # runs the tests locally
+  set +e
+  SECONDS=0
+  TEST_ARG=$@
+  echo "running test $TEST_ARG"
+  RES=$($TEST_ARG 2>&1)
+  RESPONSE_CODE=$?
+  DURATION=$SECONDS
+  if [ $RESPONSE_CODE -eq 0 ]; then
+    echo -e "${GREEN} Test \"$TEST_ARG\" successful ($DURATION sec) $NC"
+  else
+    echo -e "${RED} Test \"$TEST_ARG\" failed $NC ($DURATION sec) $NC"
+    return $RESPONSE_CODE
+  fi
+}
+
+run_in_test_repo() {
+  local test_command=$1
+
+  cd "${dir}"/test_dt_patches
+  ${test_command}
+  RESPONSE_CODE=$?
+  cd ../..
+
+  return $RESPONSE_CODE
+}
+
+test_compiler_patch() {
+  local SCALA_VERSION="$1"
+
+  run_in_test_repo "bazel build //... --repo_env=SCALA_VERSION=${SCALA_VERSION} //..."
+}
+
+#run_test_local test_compiler_patch 2.11.0
+#run_test_local test_compiler_patch 2.11.1
+#run_test_local test_compiler_patch 2.11.2
+#run_test_local test_compiler_patch 2.11.3
+#run_test_local test_compiler_patch 2.11.4
+#run_test_local test_compiler_patch 2.11.5
+#run_test_local test_compiler_patch 2.11.6
+#run_test_local test_compiler_patch 2.11.7
+#run_test_local test_compiler_patch 2.11.8
+#run_test_local test_compiler_patch 2.11.9
+#run_test_local test_compiler_patch 2.11.10
+#run_test_local test_compiler_patch 2.11.11
+run_test_local test_compiler_patch 2.11.12
+
+#run_test_local test_compiler_patch 2.12.0
+run_test_local test_compiler_patch 2.12.1
+run_test_local test_compiler_patch 2.12.2
+run_test_local test_compiler_patch 2.12.3
+run_test_local test_compiler_patch 2.12.4
+run_test_local test_compiler_patch 2.12.5
+run_test_local test_compiler_patch 2.12.6
+run_test_local test_compiler_patch 2.12.7
+run_test_local test_compiler_patch 2.12.8
+run_test_local test_compiler_patch 2.12.9
+run_test_local test_compiler_patch 2.12.10
+run_test_local test_compiler_patch 2.12.11
+run_test_local test_compiler_patch 2.12.12
+run_test_local test_compiler_patch 2.12.13
+run_test_local test_compiler_patch 2.12.14
+run_test_local test_compiler_patch 2.12.15
+run_test_local test_compiler_patch 2.12.16
+
+run_test_local test_compiler_patch 2.13.0
+run_test_local test_compiler_patch 2.13.1
+run_test_local test_compiler_patch 2.13.2
+run_test_local test_compiler_patch 2.13.3
+run_test_local test_compiler_patch 2.13.4
+run_test_local test_compiler_patch 2.13.5
+run_test_local test_compiler_patch 2.13.6
+run_test_local test_compiler_patch 2.13.7
+run_test_local test_compiler_patch 2.13.8

dt_patches/test_dt_patches/.bazelversion

@@ -0,0 +1 @@
+5.3.1

dt_patches/test_dt_patches/BUILD

@@ -0,0 +1,16 @@
+load(
+    "@io_bazel_rules_scala//scala:scala.bzl",
+    "setup_scala_toolchain",
+)
+
+SCALA_LIBS = [
+    "@scala_library",
+    "@scala_reflect",
+]
+
+setup_scala_toolchain(
+    name = "dt_scala_toolchain",
+    scala_compile_classpath = ["@scala_compiler"] + SCALA_LIBS,
+    scala_library_classpath = SCALA_LIBS,
+    scala_macro_classpath = SCALA_LIBS,
+)

dt_patches/test_dt_patches/WORKSPACE

@@ -0,0 +1,76 @@
+workspace(name = "test_dt_patches")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+skylib_version = "1.0.3"
+
+http_archive(
+    name = "bazel_skylib",
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
+    type = "tar.gz",
+    url = "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib-{}.tar.gz".format(skylib_version, skylib_version),
+)
+
+local_repository(
+    name = "io_bazel_rules_scala",
+    path = "../..",
+)
+
+load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config")
+
+scala_config(enable_compiler_dependency_tracking = True)
+
+load(
+    "@io_bazel_rules_scala//scala:scala.bzl",
+    "rules_scala_setup",
+    "rules_scala_toolchain_deps_repositories",
+)
+load(
+    "@io_bazel_rules_scala//scala:scala_cross_version.bzl",
+    "default_maven_server_urls",
+)
+load(
+    "@io_bazel_rules_scala//scala:scala_maven_import_external.bzl",
+    "scala_maven_import_external",
+)
+load("@io_bazel_rules_scala_config//:config.bzl", "SCALA_VERSION")
+
+scala_maven_import_external(
+    name = "scala_library",
+    artifact = "org.scala-lang:scala-library:%s" % SCALA_VERSION,
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "scala_compiler",
+    artifact = "org.scala-lang:scala-compiler:%s" % SCALA_VERSION,
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "scala_reflect",
+    artifact = "org.scala-lang:scala-reflect:%s" % SCALA_VERSION,
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+rules_scala_setup()
+
+rules_scala_toolchain_deps_repositories(
+    fetch_sources = True,
+    validate_scala_version = False,
+)
+
+register_toolchains(":dt_scala_toolchain")
+
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+
+rules_proto_dependencies()
+
+rules_proto_toolchains()
+
+load("@io_bazel_rules_scala//scala:toolchains.bzl", "scala_register_toolchains")
+
+scala_register_toolchains()

dt_patches/test_dt_patches/dummy/BUILD

@@ -0,0 +1,7 @@
+load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+
+scala_library(
+    name = "dummy",
+    srcs = ["Dummy.scala"],
+    visibility = ["//visibility:public"],
+)

dt_patches/test_dt_patches/dummy/Dummy.scala

@@ -0,0 +1,3 @@
+package dummy
+
+class Dummy

scala/BUILD

@@ -44,6 +44,7 @@ setup_scala_toolchain(
 
 setup_scala_toolchain(
     name = "unused_dependency_checker_error_toolchain",
+    dependency_tracking_method = "ast-plus",
     scala_compile_classpath = _SCALA_COMPILE_CLASSPATH_DEPS,
     scala_library_classpath = _SCALA_LIBRARY_CLASSPATH_DEPS,
     scala_macro_classpath = _SCALA_MACRO_CLASSPATH_DEPS,

scala/private/common_outputs.bzl

@@ -6,4 +6,5 @@ common_outputs = {
     "manifest": "%{name}_MANIFEST.MF",
     "statsfile": "%{name}.statsfile",
     "diagnosticsfile": "%{name}.diagnosticsproto",
+    "scaladepsfile": "%{name}.sdeps",
 }

scala/private/dependency.bzl

@@ -6,22 +6,19 @@ def new_dependency_info(
         dependency_mode,
         unused_deps_mode,
         strict_deps_mode,
+        compiler_deps_mode,
         dependency_tracking_method):
     is_strict_deps_on = strict_deps_mode != "off"
     is_unused_deps_on = unused_deps_mode != "off"
 
-    need_direct_jars = is_strict_deps_on or is_unused_deps_on
-    need_direct_targets = is_unused_deps_on
-
     return struct(
         dependency_mode = dependency_mode,
         need_indirect_info = is_strict_deps_on,
-        need_direct_jars = need_direct_jars,
-        need_direct_targets = need_direct_targets,
-        need_direct_info = need_direct_jars or need_direct_targets,
+        need_direct_info = is_strict_deps_on or is_unused_deps_on,
         dependency_tracking_method = dependency_tracking_method,
         unused_deps_mode = unused_deps_mode,
         strict_deps_mode = strict_deps_mode,
+        compiler_deps_mode = compiler_deps_mode,
         use_analyzer = is_strict_deps_on or is_unused_deps_on,
     )
 
@@ -31,6 +28,7 @@ def legacy_unclear_dependency_info_for_protobuf_scrooge(ctx):
         dependency_mode = _legacy_unclear_dependency_mode_for_protobuf_scrooge(ctx),
         unused_deps_mode = "off",
         strict_deps_mode = get_strict_deps_mode(ctx),
+        compiler_deps_mode = get_compiler_deps_mode(ctx),
         dependency_tracking_method = "high-level",
     )
 
@@ -47,5 +45,11 @@ def get_strict_deps_mode(ctx):
 
     return ctx.toolchains["@io_bazel_rules_scala//scala:toolchain_type"].strict_deps_mode
 
+def get_compiler_deps_mode(ctx):
+    if not hasattr(ctx.attr, "_dependency_analyzer_plugin"):
+        return "off"
+
+    return ctx.toolchains["@io_bazel_rules_scala//scala:toolchain_type"].compiler_deps_mode
+
 def _is_strict_deps_on(ctx):
     return get_strict_deps_mode(ctx) != "off"