Auto merge of #44675 - alexcrichton:many-cgu, r=aidanhs

ci: Use multiple codegen units on non-dist bots

This commit is yet another attempt to bring down our cycle times by
parallelizing some of the long-and-serial parts of the build, for example
optimizing the libsyntax, librustc, and librustc_driver crate. The hope is that
any perf loss from codegen units is more than made up for with the perf gain
from using multiple codegen units.

The value of 16 codegen units here is pretty arbitrary, it's basically just a
number which hopefully means that the cores are always nice and warm.

Author: bors

config.toml.example

@@ -223,8 +223,10 @@
 
 # Number of codegen units to use for each compiler invocation. A value of 0
 # means "the number of cores on this machine", and 1+ is passed through to the
-# compiler.
+# compiler. The `codegen-units` setting applies to std/rustc/tools whereas
+# `rustc-codegen-units` does not apply to std
 #codegen-units = 1
+#rustc-codegen-units = 1
 
 # Whether or not debug assertions are enabled for the compiler and standard
 # library. Also enables compilation of debug! and trace! logging macros.

src/bootstrap/builder.rs

@@ -458,6 +458,12 @@ impl<'a> Builder<'a> {
             stage = compiler.stage;
         }
 
+        let cgus = if mode == Mode::Libstd {
+            self.config.rust_codegen_units
+        } else {
+            self.config.rustc_codegen_units.unwrap_or(self.config.rust_codegen_units)
+        };
+
         // Customize the compiler we're running. Specify the compiler to cargo
         // as our shim and then pass it some various options used to configure
         // how the actual compiler itself is called.
@@ -468,8 +474,7 @@ impl<'a> Builder<'a> {
              .env("RUSTC", self.out.join("bootstrap/debug/rustc"))
              .env("RUSTC_REAL", self.rustc(compiler))
              .env("RUSTC_STAGE", stage.to_string())
-             .env("RUSTC_CODEGEN_UNITS",
-                  self.config.rust_codegen_units.to_string())
+             .env("RUSTC_CODEGEN_UNITS", cgus.to_string())
              .env("RUSTC_DEBUG_ASSERTIONS",
                   self.config.rust_debug_assertions.to_string())
              .env("RUSTC_SYSROOT", self.sysroot(compiler))

src/bootstrap/config.rs

@@ -82,6 +82,7 @@ pub struct Config {
     // rust codegen options
     pub rust_optimize: bool,
     pub rust_codegen_units: u32,
+    pub rustc_codegen_units: Option<u32>,
     pub rust_debug_assertions: bool,
     pub rust_debuginfo: bool,
     pub rust_debuginfo_lines: bool,
@@ -254,6 +255,7 @@ impl Default for StringOrBool {
 struct Rust {
     optimize: Option<bool>,
     codegen_units: Option<u32>,
+    rustc_codegen_units: Option<u32>,
     debug_assertions: Option<bool>,
     debuginfo: Option<bool>,
     debuginfo_lines: Option<bool>,
@@ -472,6 +474,10 @@ impl Config {
                 Some(n) => config.rust_codegen_units = n,
                 None => {}
             }
+            match rust.rustc_codegen_units {
+                Some(0) => config.rustc_codegen_units = Some(num_cpus::get() as u32),
+                other => config.rustc_codegen_units = other,
+            }
         }
 
         if let Some(ref t) = toml.target {

src/bootstrap/configure.py

@@ -256,6 +256,8 @@ def set(key, value):
                 value = True
             elif keyval[1] == "false":
                 value = False
+            elif keyval[1].isdigit():
+                value = int(keyval[1])
             else:
                 value = keyval[1]
             set(keyval[0], value)
@@ -357,6 +359,8 @@ def to_toml(value):
         return '[' + ', '.join(map(to_toml, value)) + ']'
     elif isinstance(value, str):
         return "'" + value + "'"
+    elif isinstance(value, int):
+        return str(value)
     else:
         raise 'no toml'
 

src/ci/run.sh

@@ -52,6 +52,10 @@ if [ "$DEPLOY$DEPLOY_ALT" != "" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-llvm-assertions"
   fi
 else
+  # Let's try to take advantage of some of those sweet sweet parallelism wins by
+  # using multiple codegen units during the bootstrap
+  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.rustc-codegen-units=8"
+
   # We almost always want debug assertions enabled, but sometimes this takes too
   # long for too little benefit, so we just turn them off.
   if [ "$NO_DEBUG_ASSERTIONS" = "" ]; then