Auto merge of #3409 - RalfJung:rustup, r=RalfJung

Rustup, fix rustdoc sysroot issue

This uses a different approach to resolve #3404: we entirely move the responsibility of setting miri-sysroot to whatever *invokes* the Miri driver.  cargo-miri knows whether it is inside rustdoc or not and can adjust accordingly. I previously avoided doing that because there are a bunch of places that are invoking the driver (cargo-miri, the ui test suite, `./miri run`, `./x.py run miri`) and they all need to be adjusted now. But it is also somewhat less fragile as we usually have more information there -- and we can just decide that `./miri run file.rs --sysroot path` is not supported.

Author: bors

README.md

@@ -464,10 +464,9 @@ Moreover, Miri recognizes some environment variables:
   standard library that it will build and use for interpretation. This directory
   must point to the `library` subdirectory of a `rust-lang/rust` repository
   checkout.
-* `MIRI_SYSROOT` (recognized by `cargo miri` and the Miri driver) indicates the sysroot to use. When
+* `MIRI_SYSROOT` (recognized by `cargo miri` and the test suite) indicates the sysroot to use. When
   using `cargo miri`, this skips the automatic setup -- only set this if you do not want to use the
-  automatically created sysroot. For directly invoking the Miri driver, this variable (or a
-  `--sysroot` flag) is mandatory. When invoking `cargo miri setup`, this indicates where the sysroot
+  automatically created sysroot. When invoking `cargo miri setup`, this indicates where the sysroot
   will be put.
 * `MIRI_TEST_TARGET` (recognized by the test suite and the `./miri` script) indicates which target
   architecture to test against.  `miri` and `cargo miri` accept the `--target` flag for the same

cargo-miri/src/phases.rs

@@ -168,8 +168,6 @@ pub fn phase_cargo_miri(mut args: impl Iterator<Item = String>) {
     // Forward all further arguments (not consumed by `ArgSplitFlagValue`) to cargo.
     cmd.args(args);
 
-    // Let it know where the Miri sysroot lives.
-    cmd.env("MIRI_SYSROOT", miri_sysroot);
     // Set `RUSTC_WRAPPER` to ourselves.  Cargo will prepend that binary to its usual invocation,
     // i.e., the first argument is `rustc` -- which is what we use in `main` to distinguish
     // the two codepaths. (That extra argument is why we prefer this over setting `RUSTC`.)
@@ -204,6 +202,8 @@ pub fn phase_cargo_miri(mut args: impl Iterator<Item = String>) {
     // Set rustdoc to us as well, so we can run doctests.
     cmd.env("RUSTDOC", &cargo_miri_path);
 
+    // Forward some crucial information to our own re-invocations.
+    cmd.env("MIRI_SYSROOT", miri_sysroot);
     cmd.env("MIRI_LOCAL_CRATES", local_crates(&metadata));
     if verbose > 0 {
         cmd.env("MIRI_VERBOSE", verbose.to_string()); // This makes the other phases verbose.
@@ -244,6 +244,16 @@ pub fn phase_rustc(mut args: impl Iterator<Item = String>, phase: RustcPhase) {
     /// Cargo does not give us this information directly, so we need to check
     /// various command-line flags.
     fn is_runnable_crate() -> bool {
+        // Determine whether this is cargo invoking rustc to get some infos. Ideally we'd check "is
+        // there a filename passed to rustc", but that's very hard as we would have to know whether
+        // e.g. `--print foo` is a booolean flag `--print` followed by filename `foo` or equivalent
+        // to `--print=foo`. So instead we use this more fragile approach of detecting the presence
+        // of a "query" flag rather than the absence of a filename.
+        let info_query = get_arg_flag_value("--print").is_some() || has_arg_flag("-vV");
+        if info_query {
+            // Nothing to run.
+            return false;
+        }
         let is_bin = get_arg_flag_value("--crate-type").as_deref().unwrap_or("bin") == "bin";
         let is_test = has_arg_flag("--test");
         is_bin || is_test
@@ -290,8 +300,6 @@ pub fn phase_rustc(mut args: impl Iterator<Item = String>, phase: RustcPhase) {
     let verbose = std::env::var("MIRI_VERBOSE")
         .map_or(0, |verbose| verbose.parse().expect("verbosity flag must be an integer"));
     let target_crate = is_target_crate();
-    // Determine whether this is cargo invoking rustc to get some infos.
-    let info_query = get_arg_flag_value("--print").is_some() || has_arg_flag("-vV");
 
     let store_json = |info: CrateRunInfo| {
         if get_arg_flag_value("--emit").unwrap_or_default().split(',').any(|e| e == "dep-info") {
@@ -318,7 +326,7 @@ pub fn phase_rustc(mut args: impl Iterator<Item = String>, phase: RustcPhase) {
         }
     };
 
-    let runnable_crate = !info_query && is_runnable_crate();
+    let runnable_crate = is_runnable_crate();
 
     if runnable_crate && target_crate {
         assert!(
@@ -392,7 +400,9 @@ pub fn phase_rustc(mut args: impl Iterator<Item = String>, phase: RustcPhase) {
     let mut emit_link_hack = false;
     // Arguments are treated very differently depending on whether this crate is
     // for interpretation by Miri, or for use by a build script / proc macro.
-    if !info_query && target_crate {
+    if target_crate {
+        // Set the sysroot.
+        cmd.arg("--sysroot").arg(env::var_os("MIRI_SYSROOT").unwrap());
         // Forward arguments, but remove "link" from "--emit" to make this a check-only build.
         let emit_flag = "--emit";
         while let Some(arg) = args.next() {
@@ -426,17 +436,14 @@ pub fn phase_rustc(mut args: impl Iterator<Item = String>, phase: RustcPhase) {
             cmd.arg("-C").arg("panic=abort");
         }
     } else {
-        // For host crates (but not when we are just printing some info),
-        // we might still have to set the sysroot.
-        if !info_query {
-            // When we're running `cargo-miri` from `x.py` we need to pass the sysroot explicitly
-            // due to bootstrap complications.
-            if let Some(sysroot) = std::env::var_os("MIRI_HOST_SYSROOT") {
-                cmd.arg("--sysroot").arg(sysroot);
-            }
+        // This is a host crate.
+        // When we're running `cargo-miri` from `x.py` we need to pass the sysroot explicitly
+        // due to bootstrap complications.
+        if let Some(sysroot) = std::env::var_os("MIRI_HOST_SYSROOT") {
+            cmd.arg("--sysroot").arg(sysroot);
         }
 
-        // For host crates or when we are printing, just forward everything.
+        // Forward everything.
         cmd.args(args);
     }
 
@@ -448,9 +455,7 @@ pub fn phase_rustc(mut args: impl Iterator<Item = String>, phase: RustcPhase) {
 
     // Run it.
     if verbose > 0 {
-        eprintln!(
-            "[cargo-miri rustc] target_crate={target_crate} runnable_crate={runnable_crate} info_query={info_query}"
-        );
+        eprintln!("[cargo-miri rustc] target_crate={target_crate} runnable_crate={runnable_crate}");
     }
 
     // Create a stub .rlib file if "link" was requested by cargo.
@@ -531,6 +536,12 @@ pub fn phase_runner(mut binary_args: impl Iterator<Item = String>, phase: Runner
         cmd.env(name, val);
     }
 
+    if phase != RunnerPhase::Rustdoc {
+        // Set the sysroot. Not necessary in rustdoc, where we already set the sysroot when invoking
+        // rustdoc itself, which will forward that flag when invoking rustc (i.e., us), so the flag
+        // is present in `info.args`.
+        cmd.arg("--sysroot").arg(env::var_os("MIRI_SYSROOT").unwrap());
+    }
     // Forward rustc arguments.
     // We need to patch "--extern" filenames because we forced a check-only
     // build without cargo knowing about that: replace `.rlib` suffix by
@@ -539,15 +550,13 @@ pub fn phase_runner(mut binary_args: impl Iterator<Item = String>, phase: Runner
     // but when we run here, cargo does not interpret the JSON any more. `--json`
     // then also needs to be dropped.
     let mut args = info.args.into_iter();
-    let error_format_flag = "--error-format";
-    let json_flag = "--json";
     while let Some(arg) = args.next() {
         if arg == "--extern" {
             forward_patched_extern_arg(&mut args, &mut cmd);
-        } else if let Some(suffix) = arg.strip_prefix(error_format_flag) {
+        } else if let Some(suffix) = arg.strip_prefix("--error-format") {
             assert!(suffix.starts_with('='));
             // Drop this argument.
-        } else if let Some(suffix) = arg.strip_prefix(json_flag) {
+        } else if let Some(suffix) = arg.strip_prefix("--json") {
             assert!(suffix.starts_with('='));
             // Drop this argument.
         } else {
@@ -585,13 +594,11 @@ pub fn phase_rustdoc(mut args: impl Iterator<Item = String>) {
     // just default to a straight-forward invocation for now:
     let mut cmd = Command::new("rustdoc");
 
-    let extern_flag = "--extern";
-    let runtool_flag = "--runtool";
     while let Some(arg) = args.next() {
-        if arg == extern_flag {
+        if arg == "--extern" {
             // Patch --extern arguments to use *.rmeta files, since phase_cargo_rustc only creates stub *.rlib files.
             forward_patched_extern_arg(&mut args, &mut cmd);
-        } else if arg == runtool_flag {
+        } else if arg == "--runtool" {
             // An existing --runtool flag indicates cargo is running in cross-target mode, which we don't support.
             // Note that this is only passed when cargo is run with the unstable -Zdoctest-xcompile flag;
             // otherwise, we won't be called as rustdoc at all.

miri-script/src/commands.rs

@@ -2,6 +2,7 @@ use std::env;
 use std::ffi::OsString;
 use std::io::Write;
 use std::ops::Not;
+use std::path::PathBuf;
 use std::process;
 use std::thread;
 use std::time;
@@ -20,10 +21,11 @@ const JOSH_FILTER: &str =
 const JOSH_PORT: &str = "42042";
 
 impl MiriEnv {
-    fn build_miri_sysroot(&mut self, quiet: bool) -> Result<()> {
-        if self.sh.var("MIRI_SYSROOT").is_ok() {
+    /// Returns the location of the sysroot.
+    fn build_miri_sysroot(&mut self, quiet: bool) -> Result<PathBuf> {
+        if let Some(miri_sysroot) = self.sh.var_os("MIRI_SYSROOT") {
             // Sysroot already set, use that.
-            return Ok(());
+            return Ok(miri_sysroot.into());
         }
         let manifest_path = path!(self.miri_dir / "cargo-miri" / "Cargo.toml");
         let Self { toolchain, cargo_extra_flags, .. } = &self;
@@ -57,8 +59,8 @@ impl MiriEnv {
             .with_context(|| "`cargo miri setup` failed")?;
             panic!("`cargo miri setup` didn't fail again the 2nd time?");
         };
-        self.sh.set_var("MIRI_SYSROOT", output);
-        Ok(())
+        self.sh.set_var("MIRI_SYSROOT", &output);
+        Ok(output.into())
     }
 }
 
@@ -502,7 +504,7 @@ impl Command {
             flags.iter().take_while(|arg| *arg != "--").any(|arg| *arg == "--edition");
 
         // Prepare a sysroot.
-        e.build_miri_sysroot(/* quiet */ true)?;
+        let miri_sysroot = e.build_miri_sysroot(/* quiet */ true)?;
 
         // Then run the actual command.
         let miri_manifest = path!(e.miri_dir / "Cargo.toml");
@@ -514,12 +516,12 @@ impl Command {
         if dep {
             cmd!(
                 e.sh,
-                "cargo +{toolchain} --quiet test {extra_flags...} --manifest-path {miri_manifest} --test ui -- --miri-run-dep-mode {miri_flags...} {edition_flags...} {flags...}"
+                "cargo +{toolchain} --quiet test {extra_flags...} --manifest-path {miri_manifest} --test ui -- --miri-run-dep-mode --sysroot {miri_sysroot} {miri_flags...} {edition_flags...} {flags...}"
             ).quiet().run()?;
         } else {
             cmd!(
                 e.sh,
-                "cargo +{toolchain} --quiet run {extra_flags...} --manifest-path {miri_manifest} -- {miri_flags...} {edition_flags...} {flags...}"
+                "cargo +{toolchain} --quiet run {extra_flags...} --manifest-path {miri_manifest} -- --sysroot {miri_sysroot} {miri_flags...} {edition_flags...} {flags...}"
             ).quiet().run()?;
         }
         Ok(())

rust-version

@@ -1 +1 @@
-9b8d12cf4c2311203aea83315552b15993bd4f81
+42198bf562b548015e3eae3926c175c4aabb3a7b

src/bin/miri.rs

@@ -271,25 +271,6 @@ fn run_compiler(
     callbacks: &mut (dyn rustc_driver::Callbacks + Send),
     using_internal_features: std::sync::Arc<std::sync::atomic::AtomicBool>,
 ) -> ! {
-    if target_crate {
-        // Miri needs a custom sysroot for target crates.
-        // If no `--sysroot` is given, the `MIRI_SYSROOT` env var is consulted to find where
-        // that sysroot lives, and that is passed to rustc.
-        let sysroot_flag = "--sysroot";
-        if !args.iter().any(|e| e == sysroot_flag) {
-            // Using the built-in default here would be plain wrong, so we *require*
-            // the env var to make sure things make sense.
-            let miri_sysroot = env::var("MIRI_SYSROOT").unwrap_or_else(|_| {
-                show_error!(
-                    "Miri was invoked in 'target' mode without `MIRI_SYSROOT` or `--sysroot` being set"
-                    )
-            });
-
-            args.push(sysroot_flag.to_owned());
-            args.push(miri_sysroot);
-        }
-    }
-
     // Don't insert `MIRI_DEFAULT_ARGS`, in particular, `--cfg=miri`, if we are building
     // a "host" crate. That may cause procedural macros (and probably build scripts) to
     // depend on Miri-only symbols, such as `miri_resolve_frame`:

src/shims/intrinsics/simd.rs

@@ -457,7 +457,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
                 let bitmask_len = u32::try_from(bitmask_len).unwrap();
 
                 // To read the mask, we transmute it to an integer.
-                // That does the right thing wrt endianess.
+                // That does the right thing wrt endianness.
                 let mask_ty = this.machine.layouts.uint(mask.layout.size).unwrap();
                 let mask = mask.transmute(mask_ty, this)?;
                 let mask: u64 = this.read_scalar(&mask)?.to_bits(mask_ty.size)?.try_into().unwrap();
@@ -509,7 +509,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
                     }
                 }
                 // We have to change the type of the place to be able to write `res` into it. This
-                // transmutes the integer to an array, which does the right thing wrt endianess.
+                // transmutes the integer to an array, which does the right thing wrt endianness.
                 let dest =
                     dest.transmute(this.machine.layouts.uint(dest.layout.size).unwrap(), this)?;
                 this.write_int(res, &dest)?;

src/shims/unix/linux/fd/event.rs

@@ -38,7 +38,7 @@ impl FileDescriptor for Event {
     }
 
     /// A write call adds the 8-byte integer value supplied in
-    /// its buffer (in native endianess) to the counter.  The maximum value that may be
+    /// its buffer (in native endianness) to the counter.  The maximum value that may be
     /// stored in the counter is the largest unsigned 64-bit value
     /// minus 1 (i.e., 0xfffffffffffffffe).  If the addition would
     /// cause the counter's value to exceed the maximum, then the
@@ -57,7 +57,7 @@ impl FileDescriptor for Event {
     ) -> InterpResult<'tcx, io::Result<usize>> {
         let v1 = self.val.get();
         let bytes: [u8; 8] = bytes.try_into().unwrap(); // FIXME fail gracefully when this has the wrong size
-        // Convert from target endianess to host endianess.
+        // Convert from target endianness to host endianness.
         let num = match tcx.sess.target.endian {
             Endian::Little => u64::from_le_bytes(bytes),
             Endian::Big => u64::from_be_bytes(bytes),

tests/ui.rs

@@ -54,34 +54,13 @@ fn build_so_for_c_ffi_tests() -> PathBuf {
     so_file_path
 }
 
-fn test_config(target: &str, path: &str, mode: Mode, with_dependencies: bool) -> Config {
+/// Does *not* set any args or env vars, since it is shared between the test runner and
+/// run_dep_mode.
+fn miri_config(target: &str, path: &str, mode: Mode, with_dependencies: bool) -> Config {
     // Miri is rustc-like, so we create a default builder for rustc and modify it
     let mut program = CommandBuilder::rustc();
     program.program = miri_path();
 
-    // Add some flags we always want.
-    program.args.push("-Dwarnings".into());
-    program.args.push("-Dunused".into());
-    program.args.push("-Ainternal_features".into());
-    if let Ok(extra_flags) = env::var("MIRIFLAGS") {
-        for flag in extra_flags.split_whitespace() {
-            program.args.push(flag.into());
-        }
-    }
-    program.args.push("-Zui-testing".into());
-    program.args.push("--target".into());
-    program.args.push(target.into());
-
-    // If we're on linux, and we're testing the extern-so functionality,
-    // then build the shared object file for testing external C function calls
-    // and push the relevant compiler flag.
-    if cfg!(target_os = "linux") && path.starts_with("tests/extern-so/") {
-        let so_file_path = build_so_for_c_ffi_tests();
-        let mut flag = std::ffi::OsString::from("-Zmiri-extern-so-file=");
-        flag.push(so_file_path.into_os_string());
-        program.args.push(flag);
-    }
-
     let mut config = Config {
         target: Some(target.to_owned()),
         stderr_filters: STDERR.clone(),
@@ -119,17 +98,45 @@ fn run_tests(
     with_dependencies: bool,
     tmpdir: &Path,
 ) -> Result<()> {
-    let mut config = test_config(target, path, mode, with_dependencies);
+    let mut config = miri_config(target, path, mode, with_dependencies);
 
     // Add a test env var to do environment communication tests.
     config.program.envs.push(("MIRI_ENV_VAR_TEST".into(), Some("0".into())));
-
     // Let the tests know where to store temp files (they might run for a different target, which can make this hard to find).
     config.program.envs.push(("MIRI_TEMP".into(), Some(tmpdir.to_owned().into())));
-
     // If a test ICEs, we want to see a backtrace.
     config.program.envs.push(("RUST_BACKTRACE".into(), Some("1".into())));
 
+    // Add some flags we always want.
+    config.program.args.push(
+        format!(
+            "--sysroot={}",
+            env::var("MIRI_SYSROOT").expect("MIRI_SYSROOT must be set to run the ui test suite")
+        )
+        .into(),
+    );
+    config.program.args.push("-Dwarnings".into());
+    config.program.args.push("-Dunused".into());
+    config.program.args.push("-Ainternal_features".into());
+    if let Ok(extra_flags) = env::var("MIRIFLAGS") {
+        for flag in extra_flags.split_whitespace() {
+            config.program.args.push(flag.into());
+        }
+    }
+    config.program.args.push("-Zui-testing".into());
+    config.program.args.push("--target".into());
+    config.program.args.push(target.into());
+
+    // If we're on linux, and we're testing the extern-so functionality,
+    // then build the shared object file for testing external C function calls
+    // and push the relevant compiler flag.
+    if cfg!(target_os = "linux") && path.starts_with("tests/extern-so/") {
+        let so_file_path = build_so_for_c_ffi_tests();
+        let mut flag = std::ffi::OsString::from("-Zmiri-extern-so-file=");
+        flag.push(so_file_path.into_os_string());
+        config.program.args.push(flag);
+    }
+
     // Handle command-line arguments.
     let args = ui_test::Args::test()?;
     let default_bless = env::var_os("RUSTC_BLESS").is_some_and(|v| v != "0");
@@ -292,13 +299,12 @@ fn main() -> Result<()> {
 
 fn run_dep_mode(target: String, mut args: impl Iterator<Item = OsString>) -> Result<()> {
     let path = args.next().expect("./miri run-dep must be followed by a file name");
-    let mut config = test_config(
+    let config = miri_config(
         &target,
         "",
         Mode::Yolo { rustfix: RustfixMode::Disabled },
         /* with dependencies */ true,
     );
-    config.program.args.clear(); // We want to give the user full control over flags
     let dep_args = config.build_dependencies()?;
 
     let mut cmd = config.program.build(&config.out_dir);