Hack together inline-always-overrides

Author: saethlin

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -51,7 +51,7 @@ fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage {
     }
 }
 
-fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
+fn codegen_fn_attrs_provider(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
     if cfg!(debug_assertions) {
         let def_kind = tcx.def_kind(did);
         assert!(
@@ -923,6 +923,20 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
     Some(AutoDiffAttrs { mode, width, ret_activity, input_activity: arg_activities })
 }
 
+fn codegen_fn_attrs_overridden(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
+    let mut attrs = tcx.codegen_fn_attrs_imp(did).clone();
+    let overrides = tcx.sess.opts.unstable_opts.inline_always_overrides.as_ref().unwrap();
+    if overrides.contains(&tcx.def_path_str(did)) {
+        attrs.inline = InlineAttr::Always;
+    }
+    attrs
+}
+
 pub(crate) fn provide(providers: &mut Providers) {
-    *providers = Providers { codegen_fn_attrs, should_inherit_track_caller, ..*providers };
+    *providers = Providers {
+        codegen_fn_attrs_imp: codegen_fn_attrs_provider,
+        codegen_fn_attrs_overridden,
+        should_inherit_track_caller,
+        ..*providers
+    };
 }

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -131,7 +131,7 @@ fn intern_as_new_static<'tcx>(
     // These do not inherit the codegen attrs of the parent static allocation, since
     // it doesn't make sense for them to inherit their `#[no_mangle]` and `#[link_name = ..]`
     // and the like.
-    feed.codegen_fn_attrs(CodegenFnAttrs::new());
+    feed.codegen_fn_attrs_imp(CodegenFnAttrs::new());
 
     feed.eval_static_initializer(Ok(alloc));
     feed.generics_of(tcx.generics_of(static_id).clone());

compiler/rustc_hir_analysis/src/collect.rs

@@ -313,7 +313,7 @@ impl<'tcx> Visitor<'tcx> for CollectItemTypesVisitor<'tcx> {
     fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
         if let hir::ExprKind::Closure(closure) = expr.kind {
             self.tcx.ensure_ok().generics_of(closure.def_id);
-            self.tcx.ensure_ok().codegen_fn_attrs(closure.def_id);
+            self.tcx.ensure_ok().codegen_fn_attrs_imp(closure.def_id);
             // We do not call `type_of` for closures here as that
             // depends on typecheck and would therefore hide
             // any further errors in case one typeck fails.
@@ -691,11 +691,11 @@ fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
                 }
                 match item.kind {
                     hir::ForeignItemKind::Fn(..) => {
-                        tcx.ensure_ok().codegen_fn_attrs(item.owner_id);
+                        tcx.ensure_ok().codegen_fn_attrs_imp(item.owner_id);
                         tcx.ensure_ok().fn_sig(item.owner_id)
                     }
                     hir::ForeignItemKind::Static(..) => {
-                        tcx.ensure_ok().codegen_fn_attrs(item.owner_id);
+                        tcx.ensure_ok().codegen_fn_attrs_imp(item.owner_id);
                         let mut visitor = HirPlaceholderCollector::default();
                         visitor.visit_foreign_item(item);
                         placeholder_type_error(
@@ -782,7 +782,7 @@ fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
             tcx.ensure_ok().type_of(def_id);
             tcx.ensure_ok().predicates_of(def_id);
             tcx.ensure_ok().fn_sig(def_id);
-            tcx.ensure_ok().codegen_fn_attrs(def_id);
+            tcx.ensure_ok().codegen_fn_attrs_imp(def_id);
         }
     }
 }
@@ -795,7 +795,7 @@ fn lower_trait_item(tcx: TyCtxt<'_>, trait_item_id: hir::TraitItemId) {
 
     match trait_item.kind {
         hir::TraitItemKind::Fn(..) => {
-            tcx.ensure_ok().codegen_fn_attrs(def_id);
+            tcx.ensure_ok().codegen_fn_attrs_imp(def_id);
             tcx.ensure_ok().type_of(def_id);
             tcx.ensure_ok().fn_sig(def_id);
         }
@@ -867,7 +867,7 @@ fn lower_impl_item(tcx: TyCtxt<'_>, impl_item_id: hir::ImplItemId) {
     let icx = ItemCtxt::new(tcx, def_id.def_id);
     match impl_item.kind {
         hir::ImplItemKind::Fn(..) => {
-            tcx.ensure_ok().codegen_fn_attrs(def_id);
+            tcx.ensure_ok().codegen_fn_attrs_imp(def_id);
             tcx.ensure_ok().fn_sig(def_id);
         }
         hir::ImplItemKind::Type(_) => {

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -251,7 +251,7 @@ provide! { tcx, def_id, other, cdata,
     type_alias_is_lazy => { table_direct }
     variances_of => { table }
     fn_sig => { table }
-    codegen_fn_attrs => { table }
+    codegen_fn_attrs_imp => { table }
     impl_trait_header => { table }
     const_param_default => { table }
     object_lifetime_default => { table }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1431,7 +1431,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.def_ident_span[def_id] <- ident_span);
             }
             if def_kind.has_codegen_attrs() {
-                record!(self.tables.codegen_fn_attrs[def_id] <- self.tcx.codegen_fn_attrs(def_id));
+                record!(self.tables.codegen_fn_attrs_imp[def_id] <- self.tcx.codegen_fn_attrs(def_id));
             }
             if should_encode_visibility(def_kind) {
                 let vis =

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -431,7 +431,7 @@ define_tables! {
     type_of: Table<DefIndex, LazyValue<ty::EarlyBinder<'static, Ty<'static>>>>,
     variances_of: Table<DefIndex, LazyArray<ty::Variance>>,
     fn_sig: Table<DefIndex, LazyValue<ty::EarlyBinder<'static, ty::PolyFnSig<'static>>>>,
-    codegen_fn_attrs: Table<DefIndex, LazyValue<CodegenFnAttrs>>,
+    codegen_fn_attrs_imp: Table<DefIndex, LazyValue<CodegenFnAttrs>>,
     impl_trait_header: Table<DefIndex, LazyValue<ty::ImplTraitHeader<'static>>>,
     const_param_default: Table<DefIndex, LazyValue<ty::EarlyBinder<'static, rustc_middle::ty::Const<'static>>>>,
     object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,

compiler/rustc_middle/src/query/mod.rs

@@ -1443,14 +1443,19 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query codegen_fn_attrs(def_id: DefId) -> &'tcx CodegenFnAttrs {
+    query codegen_fn_attrs_imp(def_id: DefId) -> &'tcx CodegenFnAttrs {
         desc { |tcx| "computing codegen attributes of `{}`", tcx.def_path_str(def_id) }
         arena_cache
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
         feedable
     }
 
+    query codegen_fn_attrs_overridden(def_id: DefId) -> &'tcx CodegenFnAttrs {
+        desc { |tcx| "computing codegen attributes of `{}` (with overrides)", tcx.def_path_str(def_id) }
+        arena_cache
+    }
+
     query asm_target_features(def_id: DefId) -> &'tcx FxIndexSet<Symbol> {
         desc { |tcx| "computing target features for inline asm of `{}`", tcx.def_path_str(def_id) }
     }

compiler/rustc_middle/src/ty/context.rs

@@ -3395,6 +3395,15 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn do_not_recommend_impl(self, def_id: DefId) -> bool {
         self.get_diagnostic_attr(def_id, sym::do_not_recommend).is_some()
     }
+
+    pub fn codegen_fn_attrs(self, def_id: impl IntoQueryParam<DefId>) -> &'tcx CodegenFnAttrs {
+        let def_id = def_id.into_query_param();
+        if self.sess.opts.unstable_opts.inline_always_overrides.is_some() {
+            self.codegen_fn_attrs_overridden(def_id)
+        } else {
+            self.codegen_fn_attrs_imp(def_id)
+        }
+    }
 }
 
 /// Parameter attributes that can only be determined by examining the body of a function instead

compiler/rustc_mir_transform/src/coroutine/by_move_body.rs

@@ -230,7 +230,7 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>(
     // Feed HIR because we try to access this body's attrs in the inliner.
     body_def.feed_hir();
     // Inherited from the by-ref coroutine.
-    body_def.codegen_fn_attrs(tcx.codegen_fn_attrs(coroutine_def_id).clone());
+    body_def.codegen_fn_attrs_imp(tcx.codegen_fn_attrs(coroutine_def_id).clone());
     body_def.coverage_attr_on(tcx.coverage_attr_on(coroutine_def_id));
     body_def.constness(tcx.constness(coroutine_def_id));
     body_def.coroutine_kind(tcx.coroutine_kind(coroutine_def_id));

compiler/rustc_session/src/options.rs

@@ -2249,6 +2249,9 @@ options! {
         - hashes of green query instances
         - hash collisions of query keys
         - hash collisions when creating dep-nodes"),
+    inline_always_overrides: Option<Vec<String>> = (None, parse_opt_comma_list, [TRACKED],
+        "comma-separated list of full paths to functions to treat as if they are inline(always)"
+    ),
     inline_llvm: bool = (true, parse_bool, [TRACKED],
         "enable LLVM inlining (default: yes)"),
     inline_mir: Option<bool> = (None, parse_opt_bool, [TRACKED],