Skip to content

Commit 904f591

Browse files
rust-bors[bot]rust-bors[bot]
authored
Auto merge of #142521 - sayantn:simplify-intrinsics, r=<try>
Use `LLVMIntrinsicGetDeclaration` to completely remove the hardcoded intrinsics list Follow-up to #142259 This also needs a rustc-perf run, because `Intrinsic::getType` can be expensive `@rustbot` label A-LLVM A-codegen T-compiler r? `@workingjubilee` cc `@nikic`
2 parents 49a8ba0 + 7504607 commit 904f591

File tree

6 files changed

+32
-224
lines changed

6 files changed

+32
-224
lines changed

‎compiler/rustc_codegen_llvm/src/builder.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
507507

508508
let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
509509

510-
let res = self.call_intrinsic(&name, &[self.type_ix(width)], &[lhs, rhs]);
510+
let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
511511
(self.extract_value(res, 0), self.extract_value(res, 1))
512512
}
513513

@@ -1038,7 +1038,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
10381038
let size = ty.primitive_size(self.tcx);
10391039
let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
10401040

1041-
Some(self.call_intrinsic(&name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
1041+
Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
10421042
}
10431043

10441044
/* Miscellaneous instructions */
@@ -1393,7 +1393,8 @@ impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
13931393
// Forward to the `get_static` method of `CodegenCx`
13941394
let global = self.cx().get_static(def_id);
13951395
if self.cx().tcx.is_thread_local_static(def_id) {
1396-
let pointer = self.call_intrinsic("llvm.threadlocal.address", &[], &[global]);
1396+
let pointer =
1397+
self.call_intrinsic("llvm.threadlocal.address", &[self.type_ptr()], &[global]);
13971398
// Cast to default address space if globals are in a different addrspace
13981399
self.pointercast(pointer, self.type_ptr())
13991400
} else {
@@ -1590,15 +1591,15 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
15901591
impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
15911592
pub(crate) fn call_intrinsic(
15921593
&mut self,
1593-
base_name: &str,
1594+
base_name: impl Into<Cow<'static, str>>,
15941595
type_params: &[&'ll Type],
15951596
args: &[&'ll Value],
15961597
) -> &'ll Value {
1597-
let (ty, f) = self.cx.get_intrinsic(base_name, type_params);
1598+
let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
15981599
self.call(ty, None, None, f, args, None, None)
15991600
}
16001601

1601-
fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
1602+
fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
16021603
let size = size.bytes();
16031604
if size == 0 {
16041605
return;

‎compiler/rustc_codegen_llvm/src/context.rs‎

Lines changed: 15 additions & 177 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use std::borrow::Borrow;
1+
use std::borrow::{Borrow, Cow};
22
use std::cell::{Cell, RefCell};
33
use std::ffi::{CStr, c_char, c_uint};
44
use std::marker::PhantomData;
@@ -138,7 +138,7 @@ pub(crate) struct FullCx<'ll, 'tcx> {
138138
pub rust_try_fn: Cell<Option<(&'ll Type, &'ll Value)>>,
139139

140140
intrinsics:
141-
RefCell<FxHashMap<(&'static str, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>,
141+
RefCell<FxHashMap<(Cow<'static, str>, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>,
142142

143143
/// A counter that is used for generating local symbol names
144144
local_gen_sym_counter: Cell<usize>,
@@ -845,201 +845,39 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
845845
impl<'ll> CodegenCx<'ll, '_> {
846846
pub(crate) fn get_intrinsic(
847847
&self,
848-
base_name: &str,
848+
base_name: Cow<'static, str>,
849849
type_params: &[&'ll Type],
850850
) -> (&'ll Type, &'ll Value) {
851-
if let Some(v) =
852-
self.intrinsics.borrow().get(&(base_name, SmallVec::from_slice(type_params)))
853-
{
854-
return *v;
855-
}
856-
857-
self.declare_intrinsic(base_name, type_params)
858-
}
859-
860-
fn insert_intrinsic(
861-
&self,
862-
base_name: &'static str,
863-
type_params: &[&'ll Type],
864-
args: Option<&[&'ll llvm::Type]>,
865-
ret: &'ll llvm::Type,
866-
) -> (&'ll llvm::Type, &'ll llvm::Value) {
867-
let fn_ty = if let Some(args) = args {
868-
self.type_func(args, ret)
869-
} else {
870-
self.type_variadic_func(&[], ret)
871-
};
872-
873-
let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
874-
.expect("Unknown LLVM intrinsic `{base_name}`");
875-
876-
let full_name = if intrinsic.is_overloaded() {
877-
&intrinsic.overloaded_name(self.llmod, type_params)
878-
} else {
879-
base_name
880-
};
881-
882-
let f = self.declare_cfn(full_name, llvm::UnnamedAddr::No, fn_ty);
883-
self.intrinsics
851+
*self
852+
.intrinsics
884853
.borrow_mut()
885-
.insert((base_name, SmallVec::from_slice(type_params)), (fn_ty, f));
886-
(fn_ty, f)
854+
.entry((base_name, SmallVec::from_slice(type_params)))
855+
.or_insert_with_key(|(base_name, type_params)| {
856+
self.declare_intrinsic(base_name, type_params)
857+
})
887858
}
888859

889860
fn declare_intrinsic(
890861
&self,
891862
base_name: &str,
892863
type_params: &[&'ll Type],
893864
) -> (&'ll Type, &'ll Value) {
894-
macro_rules! param {
895-
($index:literal) => {
896-
type_params[$index]
897-
};
898-
($other:expr) => {
899-
$other
900-
};
901-
}
902-
macro_rules! ifn {
903-
($name:expr, fn(...) -> $ret:expr) => (
904-
if base_name == $name {
905-
return self.insert_intrinsic($name, type_params, None, param!($ret));
906-
}
907-
);
908-
($name:expr, fn($($arg:expr),*) -> $ret:expr) => (
909-
if base_name == $name {
910-
return self.insert_intrinsic($name, type_params, Some(&[$(param!($arg)),*]), param!($ret));
911-
}
912-
);
913-
}
914-
macro_rules! mk_struct {
915-
($($field_ty:expr),*) => (self.type_struct( &[$(param!($field_ty)),*], false))
916-
}
917-
918-
let same_width_vector = |index, element_ty| {
919-
self.type_vector(element_ty, self.vector_length(type_params[index]) as u64)
920-
};
921-
922-
let ptr = self.type_ptr();
923-
let void = self.type_void();
924-
let i1 = self.type_i1();
925-
let t_i32 = self.type_i32();
926-
let t_i64 = self.type_i64();
927-
let t_isize = self.type_isize();
928-
let t_metadata = self.type_metadata();
929-
let t_token = self.type_token();
930-
931-
ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr);
932-
ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32);
933-
934-
ifn!("llvm.wasm.trunc.unsigned", fn(1) -> 0);
935-
ifn!("llvm.wasm.trunc.signed", fn(1) -> 0);
936-
ifn!("llvm.fptosi.sat", fn(1) -> 0);
937-
ifn!("llvm.fptoui.sat", fn(1) -> 0);
938-
939-
ifn!("llvm.trap", fn() -> void);
940-
ifn!("llvm.debugtrap", fn() -> void);
941-
ifn!("llvm.frameaddress", fn(t_i32) -> ptr);
942-
943-
ifn!("llvm.powi", fn(0, 1) -> 0);
944-
ifn!("llvm.pow", fn(0, 0) -> 0);
945-
ifn!("llvm.sqrt", fn(0) -> 0);
946-
ifn!("llvm.sin", fn(0) -> 0);
947-
ifn!("llvm.cos", fn(0) -> 0);
948-
ifn!("llvm.exp", fn(0) -> 0);
949-
ifn!("llvm.exp2", fn(0) -> 0);
950-
ifn!("llvm.log", fn(0) -> 0);
951-
ifn!("llvm.log10", fn(0) -> 0);
952-
ifn!("llvm.log2", fn(0) -> 0);
953-
ifn!("llvm.fma", fn(0, 0, 0) -> 0);
954-
ifn!("llvm.fmuladd", fn(0, 0, 0) -> 0);
955-
ifn!("llvm.fabs", fn(0) -> 0);
956-
ifn!("llvm.minnum", fn(0, 0) -> 0);
957-
ifn!("llvm.minimum", fn(0, 0) -> 0);
958-
ifn!("llvm.maxnum", fn(0, 0) -> 0);
959-
ifn!("llvm.maximum", fn(0, 0) -> 0);
960-
ifn!("llvm.floor", fn(0) -> 0);
961-
ifn!("llvm.ceil", fn(0) -> 0);
962-
ifn!("llvm.trunc", fn(0) -> 0);
963-
ifn!("llvm.copysign", fn(0, 0) -> 0);
964-
ifn!("llvm.round", fn(0) -> 0);
965-
ifn!("llvm.rint", fn(0) -> 0);
966-
ifn!("llvm.nearbyint", fn(0) -> 0);
967-
968-
ifn!("llvm.ctpop", fn(0) -> 0);
969-
ifn!("llvm.ctlz", fn(0, i1) -> 0);
970-
ifn!("llvm.cttz", fn(0, i1) -> 0);
971-
ifn!("llvm.bswap", fn(0) -> 0);
972-
ifn!("llvm.bitreverse", fn(0) -> 0);
973-
ifn!("llvm.fshl", fn(0, 0, 0) -> 0);
974-
ifn!("llvm.fshr", fn(0, 0, 0) -> 0);
975-
976-
ifn!("llvm.sadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
977-
ifn!("llvm.uadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
978-
ifn!("llvm.ssub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
979-
ifn!("llvm.usub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
980-
ifn!("llvm.smul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
981-
ifn!("llvm.umul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
982-
983-
ifn!("llvm.sadd.sat", fn(0, 0) -> 0);
984-
ifn!("llvm.uadd.sat", fn(0, 0) -> 0);
985-
ifn!("llvm.ssub.sat", fn(0, 0) -> 0);
986-
ifn!("llvm.usub.sat", fn(0, 0) -> 0);
987-
988-
ifn!("llvm.scmp", fn(1, 1) -> 0);
989-
ifn!("llvm.ucmp", fn(1, 1) -> 0);
990-
991-
ifn!("llvm.lifetime.start", fn(t_i64, 0) -> void);
992-
ifn!("llvm.lifetime.end", fn(t_i64, 0) -> void);
993-
994-
ifn!("llvm.is.constant", fn(0) -> i1);
995-
ifn!("llvm.expect", fn(0, 0) -> 0);
996-
997-
ifn!("llvm.eh.typeid.for", fn(0) -> t_i32);
998-
ifn!("llvm.localescape", fn(...) -> void);
999-
ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr);
1000-
1001-
ifn!("llvm.assume", fn(i1) -> void);
1002-
ifn!("llvm.prefetch", fn(0, t_i32, t_i32, t_i32) -> void);
1003-
1004865
// This isn't an "LLVM intrinsic", but LLVM's optimization passes
1005866
// recognize it like one (including turning it into `bcmp` sometimes)
1006867
// and we use it to implement intrinsics like `raw_eq` and `compare_bytes`
1007868
if base_name == "memcmp" {
1008-
let fn_ty = self.type_func(&[ptr, ptr, t_isize], self.type_int());
869+
let fn_ty = self
870+
.type_func(&[self.type_ptr(), self.type_ptr(), self.type_isize()], self.type_int());
1009871
let f = self.declare_cfn("memcmp", llvm::UnnamedAddr::No, fn_ty);
1010-
self.intrinsics.borrow_mut().insert(("memcmp", SmallVec::new()), (fn_ty, f));
1011872

1012873
return (fn_ty, f);
1013874
}
1014875

1015-
// variadic intrinsics
1016-
ifn!("llvm.va_start", fn(0) -> void);
1017-
ifn!("llvm.va_end", fn(0) -> void);
1018-
ifn!("llvm.va_copy", fn(0, 0) -> void);
1019-
1020-
if self.sess().instrument_coverage() {
1021-
ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void);
1022-
ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void);
1023-
ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void);
1024-
}
1025-
1026-
ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1);
1027-
ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1});
1028-
1029-
if self.sess().opts.debuginfo != DebugInfo::None {
1030-
ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata, t_metadata) -> void);
1031-
ifn!("llvm.dbg.value", fn(t_metadata, t_metadata, t_metadata) -> void);
1032-
}
1033-
1034-
ifn!("llvm.ptrmask", fn(0, 1) -> 0);
1035-
ifn!("llvm.threadlocal.address", fn(ptr) -> ptr);
1036-
1037-
ifn!("llvm.masked.load", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
1038-
ifn!("llvm.masked.store", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
1039-
ifn!("llvm.masked.gather", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
1040-
ifn!("llvm.masked.scatter", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
876+
let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
877+
.expect(&format!("Unknown intrinsic: `{base_name}`"));
878+
let f = intrinsic.get_declaration(self.llmod, &type_params);
1041879

1042-
bug!("Unknown intrinsic: `{base_name}`")
880+
(self.get_type_of_global(f), f)
1043881
}
1044882

1045883
pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value {

‎compiler/rustc_codegen_llvm/src/intrinsic.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
378378
sym::ctlz | sym::cttz => {
379379
let y = self.const_bool(false);
380380
let ret = self.call_intrinsic(
381-
&format!("llvm.{name}"),
381+
format!("llvm.{name}"),
382382
&[llty],
383383
&[args[0].immediate(), y],
384384
);
@@ -423,7 +423,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
423423
// always uses `u32`.
424424
let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
425425

426-
self.call_intrinsic(&llvm_name, &[llty], &[val, val, raw_shift])
426+
self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift])
427427
}
428428
sym::saturating_add | sym::saturating_sub => {
429429
let is_add = name == sym::saturating_add;
@@ -434,7 +434,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
434434
if signed { 's' } else { 'u' },
435435
if is_add { "add" } else { "sub" },
436436
);
437-
self.call_intrinsic(&llvm_name, &[llty], &[lhs, rhs])
437+
self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
438438
}
439439
_ => bug!(),
440440
}
@@ -2393,7 +2393,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
23932393
);
23942394
let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
23952395

2396-
return Ok(bx.call_intrinsic(&llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
2396+
return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
23972397
}
23982398

23992399
span_bug!(span, "unknown SIMD intrinsic");

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1078,8 +1078,6 @@ unsafe extern "C" {
10781078

10791079
// Operations on other types
10801080
pub(crate) fn LLVMVoidTypeInContext(C: &Context) -> &Type;
1081-
pub(crate) fn LLVMTokenTypeInContext(C: &Context) -> &Type;
1082-
pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type;
10831081

10841082
// Operations on all values
10851083
pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type;
@@ -1198,14 +1196,12 @@ unsafe extern "C" {
11981196

11991197
// Operations about llvm intrinsics
12001198
pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint;
1201-
pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero<c_uint>) -> Bool;
1202-
pub(crate) fn LLVMIntrinsicCopyOverloadedName2<'a>(
1199+
pub(crate) fn LLVMGetIntrinsicDeclaration<'a>(
12031200
Mod: &'a Module,
12041201
ID: NonZero<c_uint>,
12051202
ParamTypes: *const &'a Type,
12061203
ParamCount: size_t,
1207-
NameLength: *mut size_t,
1208-
) -> *mut c_char;
1204+
) -> &'a Value;
12091205

12101206
// Operations on parameters
12111207
pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;

‎compiler/rustc_codegen_llvm/src/llvm/mod.rs‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
use std::ffi::{CStr, CString};
44
use std::num::NonZero;
5+
use std::ptr;
56
use std::str::FromStr;
67
use std::string::FromUtf8Error;
7-
use std::{ptr, slice};
88

99
use libc::c_uint;
1010
use rustc_abi::{Align, Size, WrappingRange};
@@ -339,34 +339,14 @@ impl Intrinsic {
339339
NonZero::new(id).map(|id| Self { id })
340340
}
341341

342-
pub(crate) fn is_overloaded(self) -> bool {
343-
unsafe { LLVMIntrinsicIsOverloaded(self.id) == True }
344-
}
345-
346-
pub(crate) fn overloaded_name<'ll>(
342+
pub(crate) fn get_declaration<'ll>(
347343
self,
348344
llmod: &'ll Module,
349345
type_params: &[&'ll Type],
350-
) -> String {
351-
let mut len = 0;
352-
let ptr = unsafe {
353-
LLVMIntrinsicCopyOverloadedName2(
354-
llmod,
355-
self.id,
356-
type_params.as_ptr(),
357-
type_params.len(),
358-
&mut len,
359-
)
360-
};
361-
362-
let slice = unsafe { slice::from_raw_parts_mut(ptr.cast(), len) };
363-
let copied = str::from_utf8(slice).expect("Non-UTF8 intrinsic name").to_string();
364-
346+
) -> &'ll Value {
365347
unsafe {
366-
libc::free(ptr.cast());
348+
LLVMGetIntrinsicDeclaration(llmod, self.id, type_params.as_ptr(), type_params.len())
367349
}
368-
369-
copied
370350
}
371351
}
372352

0 commit comments

Comments
 (0)