strip prefix of temporary file names when it exceeds filesystem name length limit

compiler/rustc_session/src/config.rs

@@ -16,10 +16,11 @@ use std::{cmp, fmt, fs, iter};
 
 use externs::{ExternOpt, split_extern_opt};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
-use rustc_data_structures::stable_hasher::{StableOrd, ToStableHashKey};
+use rustc_data_structures::stable_hasher::{StableHasher, StableOrd, ToStableHashKey};
 use rustc_errors::emitter::HumanReadableErrorType;
 use rustc_errors::{ColorConfig, DiagArgValue, DiagCtxtFlags, IntoDiagArg};
 use rustc_feature::UnstableFeatures;
+use rustc_hashes::Hash64;
 use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 use rustc_span::edition::{DEFAULT_EDITION, EDITION_NAME_LIST, Edition, LATEST_STABLE_EDITION};
 use rustc_span::source_map::FilePathMapping;
@@ -1198,7 +1199,25 @@ pub struct OutputFilenames {
 pub const RLINK_EXT: &str = "rlink";
 pub const RUST_CGU_EXT: &str = "rcgu";
 pub const DWARF_OBJECT_EXT: &str = "dwo";
+pub const MAX_FILENAME_LENGTH: usize = 143; // ecryptfs limits filenames to 143 bytes see #49914
 
+/// Ensure the filename is not too long, as some filesystems have a limit.
+/// If the filename is too long, hash part of it and append the hash to the filename.
+/// This is a workaround for long crate names generating overly long filenames.
+fn maybe_strip_file_name(mut path: PathBuf) -> PathBuf {
+    if path.file_name().map_or(0, |name| name.len()) > MAX_FILENAME_LENGTH {
+        let filename = path.file_name().unwrap().to_string_lossy();
+        let hash_len = 64 / 4; // Hash64 is 64 bits encoded in hex
+        let stripped_len = filename.len() - MAX_FILENAME_LENGTH + hash_len;
+
+        let mut hasher = StableHasher::new();
+        filename[..stripped_len].hash(&mut hasher);
+        let hash = hasher.finish::<Hash64>();
+
+        path.set_file_name(format!("{:x}-{}", hash, &filename[stripped_len..]));
+    }
+    path
+}
 impl OutputFilenames {
     pub fn new(
         out_directory: PathBuf,
@@ -1291,7 +1310,7 @@ impl OutputFilenames {
         }
 
         let temps_directory = self.temps_directory.as_ref().unwrap_or(&self.out_directory);
-        self.with_directory_and_extension(temps_directory, &extension)
+        maybe_strip_file_name(self.with_directory_and_extension(temps_directory, &extension))
     }
 
     pub fn temp_path_for_diagnostic(&self, ext: &str) -> PathBuf {

tests/run-make/lto-long-filenames/main.rs

@@ -0,0 +1,7 @@
+// This file has very long lines, but there is no way to avoid it as we are testing
+// long crate names. so:
+// ignore-tidy-linelength
+
+extern crate generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name;
+
+fn main() {}

tests/run-make/lto-long-filenames/rmake.rs

@@ -0,0 +1,32 @@
+// This file has very long lines, but there is no way to avoid it as we are testing
+// long crate names. so:
+// ignore-tidy-linelength
+
+// A variant of the smoke test to check that link time optimization
+// (LTO) is accepted by the compiler, and that
+// passing its various flags still results in successful compilation, even for very long crate names.
+// See https://github.com/rust-lang/rust/issues/49914
+
+//@ ignore-cross-compile
+
+use std::fs;
+
+use run_make_support::{rfs, rustc};
+
+// This test make sure we don't get such following error:
+// error: could not write output to generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.9384edb61bfd127c-cgu.0.rcgu.o: File name too long
+// as reported in issue #49914
+fn main() {
+    let lto_flags = ["-Clto", "-Clto=yes", "-Clto=off", "-Clto=thin", "-Clto=fat"];
+    let aux_file = "generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.rs";
+    // The auxiliary file is used to test long crate names.
+    // The file name is intentionally long to test the handling of long filenames.
+    // We don't commit it to avoid issues with Windows paths which have known limitations for the full path length.
+    // Posix usually only have a limit for the length of the file name.
+    rfs::write(aux_file, "#![crate_type = \"rlib\"]\n");
+
+    for flag in lto_flags {
+        rustc().input(aux_file).arg(flag).run();
+        rustc().input("main.rs").arg(flag).run();
+    }
+}