Auto merge of #105601 - BelovDV:change-rlib-with-not-stable, r=petrochenkov

Enable new rlib in non stable cases

If bundled static library uses cfg (unstable) or whole-archive (wasn't supported) bundled libs are packed even without packed_bundled_libs.

r? `@petrochenkov`

Author: bors

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -270,10 +270,9 @@ pub fn each_linked_rlib(
 
 /// Create an 'rlib'.
 ///
-/// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains
-/// the object file of the crate, but it also contains all of the object files from native
-/// libraries. This is done by unzipping native libraries and inserting all of the contents into
-/// this archive.
+/// An rlib in its current incarnation is essentially a renamed .a file (with "dummy" object files).
+/// The rlib primarily contains the object file of the crate, but it also some of the object files
+/// from native libraries.
 fn link_rlib<'a>(
     sess: &'a Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
@@ -347,44 +346,23 @@ fn link_rlib<'a>(
     // loaded from the libraries found here and then encode that into the
     // metadata of the rlib we're generating somehow.
     for lib in codegen_results.crate_info.used_libraries.iter() {
-        match lib.kind {
-            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
-                if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.packed_bundled_libs => {}
-            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
-                if flavor == RlibFlavor::Normal =>
-            {
-                // Don't allow mixing +bundle with +whole_archive since an rlib may contain
-                // multiple native libs, some of which are +whole-archive and some of which are
-                // -whole-archive and it isn't clear how we can currently handle such a
-                // situation correctly.
-                // See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897
-                sess.emit_err(errors::IncompatibleLinkingModifiers);
-            }
-            NativeLibKind::Static { bundle: None | Some(true), .. } => {}
-            NativeLibKind::Static { bundle: Some(false), .. }
-            | NativeLibKind::Dylib { .. }
-            | NativeLibKind::Framework { .. }
-            | NativeLibKind::RawDylib
-            | NativeLibKind::LinkArg
-            | NativeLibKind::Unspecified => continue,
-        }
-        if let Some(name) = lib.name {
-            let location =
+        let NativeLibKind::Static { bundle: None | Some(true), whole_archive } = lib.kind else {
+            continue;
+        };
+        if whole_archive == Some(true) && !codegen_results.crate_info.feature_packed_bundled_libs {
+            sess.emit_err(errors::IncompatibleLinkingModifiers);
+        }
+        if flavor == RlibFlavor::Normal && let Some(filename) = lib.filename {
+            let path = find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
+            let src = read(path).map_err(|e| sess.emit_fatal(errors::ReadFileError {message: e }))?;
+            let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
+            let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
+            packed_bundled_libs.push(wrapper_file);
+        } else if let Some(name) = lib.name {
+            let path =
                 find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
-            if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal {
-                let filename = lib.filename.unwrap();
-                let lib_path =
-                    find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
-                let src = read(lib_path)
-                    .map_err(|e| sess.emit_fatal(errors::ReadFileError { message: e }))?;
-                let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
-                let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
-                packed_bundled_libs.push(wrapper_file);
-                continue;
-            }
-            ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|error| {
-                sess.emit_fatal(errors::AddNativeLibrary { library_path: location, error });
-            });
+            ab.add_archive(&path, Box::new(|_| false)).unwrap_or_else(|error| {
+                sess.emit_fatal(errors::AddNativeLibrary { library_path: path, error })});
         }
     }
 
@@ -516,36 +494,14 @@ fn link_staticlib<'a>(
         &codegen_results.crate_info,
         Some(CrateType::Staticlib),
         &mut |cnum, path| {
-            let name = codegen_results.crate_info.crate_name[&cnum];
-            let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
-
-            // Here when we include the rlib into our staticlib we need to make a
-            // decision whether to include the extra object files along the way.
-            // These extra object files come from statically included native
-            // libraries, but they may be cfg'd away with #[link(cfg(..))].
-            //
-            // This unstable feature, though, only needs liblibc to work. The only
-            // use case there is where musl is statically included in liblibc.rlib,
-            // so if we don't want the included version we just need to skip it. As
-            // a result the logic here is that if *any* linked library is cfg'd away
-            // we just skip all object files.
-            //
-            // Clearly this is not sufficient for a general purpose feature, and
-            // we'd want to read from the library's metadata to determine which
-            // object files come from where and selectively skip them.
-            let skip_object_files = native_libs.iter().any(|lib| {
-                matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
-                    && !relevant_lib(sess, lib)
-            });
-
             let lto = are_upstream_rust_objects_already_included(sess)
                 && !ignored_for_lto(sess, &codegen_results.crate_info, cnum);
 
-            // Ignoring obj file starting with the crate name
-            // as simple comparison is not enough - there
-            // might be also an extra name suffix
-            let obj_start = name.as_str().to_owned();
+            let native_libs = codegen_results.crate_info.native_libraries[&cnum].iter();
+            let relevant = native_libs.clone().filter(|lib| relevant_lib(sess, &lib));
+            let relevant_libs: FxHashSet<_> = relevant.filter_map(|lib| lib.filename).collect();
 
+            let bundled_libs: FxHashSet<_> = native_libs.filter_map(|lib| lib.filename).collect();
             ab.add_archive(
                 path,
                 Box::new(move |fname: &str| {
@@ -559,20 +515,25 @@ fn link_staticlib<'a>(
                         return true;
                     }
 
-                    // Otherwise if this is *not* a rust object and we're skipping
-                    // objects then skip this file
-                    if skip_object_files
-                        && (!fname.starts_with(&obj_start) || !fname.ends_with(".o"))
-                    {
+                    // Skip objects for bundled libs.
+                    if bundled_libs.contains(&Symbol::intern(fname)) {
                         return true;
                     }
 
-                    // ok, don't skip this
                     false
                 }),
             )
             .unwrap();
 
+            archive_builder_builder
+                .extract_bundled_libs(path, tempdir.as_ref(), &relevant_libs)
+                .unwrap_or_else(|e| sess.emit_fatal(e));
+            for filename in relevant_libs {
+                let joined = tempdir.as_ref().join(filename.as_str());
+                let path = joined.as_path();
+                ab.add_archive(path, Box::new(|_| false)).unwrap();
+            }
+
             all_native_libs
                 .extend(codegen_results.crate_info.native_libraries[&cnum].iter().cloned());
         },
@@ -2590,18 +2551,8 @@ fn add_static_crate<'a>(
         cmd.link_rlib(&fix_windows_verbatim_for_gcc(path));
     };
 
-    // See the comment above in `link_staticlib` and `link_rlib` for why if
-    // there's a static library that's not relevant we skip all object
-    // files.
-    let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
-    let skip_native = native_libs.iter().any(|lib| {
-        matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
-            && !relevant_lib(sess, lib)
-    });
-
-    if (!are_upstream_rust_objects_already_included(sess)
-        || ignored_for_lto(sess, &codegen_results.crate_info, cnum))
-        && !skip_native
+    if !are_upstream_rust_objects_already_included(sess)
+        || ignored_for_lto(sess, &codegen_results.crate_info, cnum)
     {
         link_upstream(cratepath);
         return;
@@ -2632,17 +2583,13 @@ fn add_static_crate<'a>(
                 let is_rust_object =
                     canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f);
 
-                // If we've been requested to skip all native object files
-                // (those not generated by the rust compiler) then we can skip
-                // this file. See above for why we may want to do this.
-                let skip_because_cfg_say_so = skip_native && !is_rust_object;
-
                 // If we're performing LTO and this is a rust-generated object
                 // file, then we don't need the object file as it's part of the
                 // LTO module. Note that `#![no_builtins]` is excluded from LTO,
                 // though, so we let that object file slide.
-                let skip_because_lto =
-                    upstream_rust_objects_already_included && is_rust_object && is_builtins;
+                if upstream_rust_objects_already_included && is_rust_object && is_builtins {
+                    return true;
+                }
 
                 // We skip native libraries because:
                 // 1. This native libraries won't be used from the generated rlib,
@@ -2653,10 +2600,6 @@ fn add_static_crate<'a>(
                     return true;
                 }
 
-                if skip_because_cfg_say_so || skip_because_lto {
-                    return true;
-                }
-
                 false
             }),
         ) {

compiler/rustc_codegen_ssa/src/base.rs

@@ -858,6 +858,7 @@ impl CrateInfo {
             dependency_formats: tcx.dependency_formats(()).clone(),
             windows_subsystem,
             natvis_debugger_visualizers: Default::default(),
+            feature_packed_bundled_libs: tcx.features().packed_bundled_libs,
         };
         let crates = tcx.crates(());
 

compiler/rustc_codegen_ssa/src/lib.rs

@@ -159,6 +159,7 @@ pub struct CrateInfo {
     pub dependency_formats: Lrc<Dependencies>,
     pub windows_subsystem: Option<String>,
     pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>,
+    pub feature_packed_bundled_libs: bool, // unstable feature flag.
 }
 
 #[derive(Encodable, Decodable)]

compiler/rustc_error_messages/locales/en-US/codegen_ssa.ftl

@@ -22,7 +22,7 @@ codegen_ssa_ignoring_output = ignoring -o because multiple .{$extension} files w
 
 codegen_ssa_create_temp_dir = couldn't create a temp dir: {$error}
 
-codegen_ssa_incompatible_linking_modifiers = the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
+codegen_ssa_incompatible_linking_modifiers = link modifiers combination `+bundle,+whole-archive` is unstable when generating rlibs
 
 codegen_ssa_add_native_library = failed to add native library {$library_path}: {$error}
 

compiler/rustc_feature/src/active.rs

@@ -164,6 +164,8 @@ declare_features! (
     (active, multiple_supertrait_upcastable, "CURRENT_RUSTC_VERSION", None, None),
     /// Allows using `#[omit_gdb_pretty_printer_section]`.
     (active, omit_gdb_pretty_printer_section, "1.5.0", None, None),
+    /// Allows using `+bundled,+whole-archive` native libs.
+    (active, packed_bundled_libs, "1.67.0", None, None),
     /// Allows using `#[prelude_import]` on glob `use` items.
     (active, prelude_import, "1.2.0", None, None),
     /// Used to identify crates that contain the profiler runtime.

compiler/rustc_metadata/src/native_libs.rs

@@ -49,20 +49,21 @@ fn find_bundled_library(
     name: Option<Symbol>,
     verbatim: Option<bool>,
     kind: NativeLibKind,
+    has_cfg: bool,
     sess: &Session,
 ) -> Option<Symbol> {
-    if sess.opts.unstable_opts.packed_bundled_libs &&
-            sess.crate_types().iter().any(|ct| ct == &CrateType::Rlib || ct == &CrateType::Staticlib) &&
-            let NativeLibKind::Static { bundle: Some(true) | None, .. } = kind {
-        find_native_static_library(
-            name.unwrap().as_str(),
-            verbatim.unwrap_or(false),
-            &sess.target_filesearch(PathKind::Native).search_path_dirs(),
-            sess,
-        ).file_name().and_then(|s| s.to_str()).map(Symbol::intern)
-    } else {
-        None
+    if let NativeLibKind::Static { bundle: Some(true) | None, whole_archive } = kind
+        && sess.crate_types().iter().any(|t| matches!(t, &CrateType::Rlib | CrateType::Staticlib))
+        && (sess.opts.unstable_opts.packed_bundled_libs || has_cfg || whole_archive == Some(true))
+    {
+        let verbatim = verbatim.unwrap_or(false);
+        let search_paths = &sess.target_filesearch(PathKind::Native).search_path_dirs();
+        return find_native_static_library(name.unwrap().as_str(), verbatim, search_paths, sess)
+            .file_name()
+            .and_then(|s| s.to_str())
+            .map(Symbol::intern);
     }
+    None
 }
 
 pub(crate) fn collect(tcx: TyCtxt<'_>) -> Vec<NativeLib> {
@@ -385,7 +386,7 @@ impl<'tcx> Collector<'tcx> {
 
             let name = name.map(|(name, _)| name);
             let kind = kind.unwrap_or(NativeLibKind::Unspecified);
-            let filename = find_bundled_library(name, verbatim, kind, sess);
+            let filename = find_bundled_library(name, verbatim, kind, cfg.is_some(), sess);
             self.libs.push(NativeLib {
                 name,
                 filename,
@@ -475,7 +476,7 @@ impl<'tcx> Collector<'tcx> {
                 let name = Some(Symbol::intern(new_name.unwrap_or(&passed_lib.name)));
                 let sess = self.tcx.sess;
                 let filename =
-                    find_bundled_library(name, passed_lib.verbatim, passed_lib.kind, sess);
+                    find_bundled_library(name, passed_lib.verbatim, passed_lib.kind, false, sess);
                 self.libs.push(NativeLib {
                     name,
                     filename,

compiler/rustc_span/src/symbol.rs

@@ -1049,6 +1049,7 @@ symbols! {
         overlapping_marker_traits,
         owned_box,
         packed,
+        packed_bundled_libs,
         panic,
         panic_2015,
         panic_2021,

tests/run-make/rlib-format-packed-bundled-libs-3/Makefile

@@ -0,0 +1,35 @@
+-include ../../run-make-fulldeps/tools.mk
+
+# ignore-cross-compile
+# only-linux
+
+# Make sure -Zpacked_bundled_libs-like behavior activates with whole-archive.
+
+# We're using the llvm-nm instead of the system nm to ensure it is compatible
+# with the LLVM bitcode generated by rustc.
+NM = "$(LLVM_BIN_DIR)"/llvm-nm
+
+all: $(call NATIVE_STATICLIB,native_dep_1) $(call NATIVE_STATICLIB,native_dep_2) $(call NATIVE_STATICLIB,native_dep_3) $(call NATIVE_STATICLIB,native_dep_4)
+	# test cfg with packed bundle
+	$(RUSTC) rust_dep_cfg.rs --crate-type=rlib -Zpacked_bundled_libs
+	$(RUSTC) main.rs --extern rust_dep=$(TMPDIR)/librust_dep_cfg.rlib --crate-type=staticlib --cfg should_add
+	$(AR) t $(TMPDIR)/librust_dep_cfg.rlib | $(CGREP) -e "libnative_dep_1.a"
+	$(AR) t $(TMPDIR)/librust_dep_cfg.rlib | $(CGREP) -e "libnative_dep_2.a"
+	$(AR) t $(TMPDIR)/libmain.a | $(CGREP) -e "libnative_dep_1.o"
+	$(AR) t $(TMPDIR)/libmain.a | $(CGREP) -ev "libnative_dep_2.o"
+
+
+	# test bundle with whole_archive
+	$(RUSTC) rust_dep.rs --crate-type=rlib
+	$(AR) t $(TMPDIR)/librust_dep.rlib | $(CGREP) -e "native_dep_1"
+	$(AR) t $(TMPDIR)/librust_dep.rlib | $(CGREP) -e "native_dep_3"
+	$(AR) t $(TMPDIR)/librust_dep.rlib | $(CGREP) -ev "native_dep_2"
+	$(AR) t $(TMPDIR)/librust_dep.rlib | $(CGREP) -ev "native_dep_4"
+
+	# Make sure compiler doesn't use files, that it shouldn't know about.
+	rm $(TMPDIR)/libnative_dep_1.a
+	rm $(TMPDIR)/libnative_dep_3.a
+
+	$(RUSTC) main.rs --extern rust_dep=$(TMPDIR)/librust_dep.rlib --print link-args > $(TMPDIR)/link_args
+	cat $(TMPDIR)/link_args | $(CGREP) -ev "native_dep_3"
+	cat $(TMPDIR)/link_args | $(CGREP) -e "--whole-archive.*native_dep_1.*--whole-archive.*lnative_dep_2.*no-whole-archive.*lnative_dep_4"

tests/run-make/rlib-format-packed-bundled-libs-3/main.rs

@@ -0,0 +1,5 @@
+extern crate rust_dep;
+
+pub fn main() {
+    rust_dep::rust_dep();
+}

tests/run-make/rlib-format-packed-bundled-libs-3/native_dep_1.c

@@ -0,0 +1 @@
+int native_f1() { return 1; }