Implement vconst instruction for x86

cranelift-codegen/meta/src/isa/x86/encodings.rs

@@ -270,7 +270,7 @@ impl PerCpuModeEncodings {
     /// Add the same encoding to both X86_32 and X86_64; assumes configuration (e.g. REX, operand binding) has already happened
     fn enc_32_64_maybe_isap(
         &mut self,
-        inst: BoundInstruction,
+        inst: impl Clone + Into<InstSpec>,
         template: Template,
         isap: Option<SettingPredicateNumber>,
     ) {
@@ -280,7 +280,7 @@ impl PerCpuModeEncodings {
 
     fn enc32_maybe_isap(
         &mut self,
-        inst: BoundInstruction,
+        inst: impl Into<InstSpec>,
         template: Template,
         isap: Option<SettingPredicateNumber>,
     ) {
@@ -292,7 +292,7 @@ impl PerCpuModeEncodings {
 
     fn enc64_maybe_isap(
         &mut self,
-        inst: BoundInstruction,
+        inst: impl Into<InstSpec>,
         template: Template,
         isap: Option<SettingPredicateNumber>,
     ) {
@@ -432,6 +432,7 @@ pub fn define(
     let uload8_complex = shared.by_name("uload8_complex");
     let ushr = shared.by_name("ushr");
     let ushr_imm = shared.by_name("ushr_imm");
+    let vconst = shared.by_name("vconst");
     let x86_bsf = x86.by_name("x86_bsf");
     let x86_bsr = x86.by_name("x86_bsr");
     let x86_cvtt2si = x86.by_name("x86_cvtt2si");
@@ -578,6 +579,7 @@ pub fn define(
     let rec_urm = r.template("urm");
     let rec_urm_noflags = r.template("urm_noflags");
     let rec_urm_noflags_abcd = r.template("urm_noflags_abcd");
+    let rec_vconst = r.template("vconst");
 
     // Predicates shorthands.
     let all_ones_funcaddrs_and_not_is_pic =
@@ -1785,6 +1787,18 @@ pub fn define(
         }
     }
 
+    // SIMD vconst using MOVUPS
+    // TODO it would be ideal if eventually this became the more efficient MOVAPS but we would have
+    // to guarantee that the constants are aligned when emitted and there is currently no mechanism
+    // for that; alternately, constants could be loaded into XMM registers using a sequence like:
+    // MOVQ + MOVHPD + MOVQ + MOVLPD (this allows the constants to be immediates instead of stored
+    // in memory) but some performance measurements are needed.
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() >= 8) {
+        let instruction = vconst.bind_vector_from_lane(ty, sse_vector_size);
+        let template = rec_vconst.nonrex().opcodes(vec![0x0f, 0x10]);
+        e.enc_32_64_maybe_isap(instruction, template, None); // from SSE
+    }
+
     // Reference type instructions
 
     // Null references implemented as iconst 0.

cranelift-codegen/meta/src/isa/x86/recipes.rs

@@ -399,6 +399,7 @@ pub fn define<'shared>(
     let f_unary_ieee32 = formats.by_name("UnaryIeee32");
     let f_unary_ieee64 = formats.by_name("UnaryIeee64");
     let f_unary_imm = formats.by_name("UnaryImm");
+    let f_unary_imm128 = formats.by_name("UnaryImm128");
 
     // Predicates shorthands.
     let use_sse41 = settings.predicate_by_name("use_sse41");
@@ -2382,6 +2383,19 @@ pub fn define<'shared>(
             ),
     );
 
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("vconst", f_unary_imm128, 5)
+            .operands_out(vec![fpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(0, out_reg0), sink);
+                    modrm_riprel(out_reg0, sink);
+                    const_disp4(imm, func, sink);
+                "#,
+            ),
+    );
+
     recipes.add_template_recipe(
         EncodingRecipeBuilder::new("jt_base", f_branch_table_base, 5)
             .operands_out(vec![gpr])

cranelift-codegen/meta/src/shared/formats.rs

@@ -5,6 +5,7 @@ pub fn define(immediates: &OperandKinds, entities: &OperandKinds) -> FormatRegis
     // Shorthands for immediates.
     let uimm8 = immediates.by_name("uimm8");
     let uimm32 = immediates.by_name("uimm32");
+    let uimm128 = immediates.by_name("uimm128");
     let imm64 = immediates.by_name("imm64");
     let ieee32 = immediates.by_name("ieee32");
     let ieee64 = immediates.by_name("ieee64");
@@ -30,6 +31,7 @@ pub fn define(immediates: &OperandKinds, entities: &OperandKinds) -> FormatRegis
 
     registry.insert(Builder::new("Unary").value());
     registry.insert(Builder::new("UnaryImm").imm(imm64));
+    registry.insert(Builder::new("UnaryImm128").imm(uimm128));
     registry.insert(Builder::new("UnaryIeee32").imm(ieee32));
     registry.insert(Builder::new("UnaryIeee64").imm(ieee64));
     registry.insert(Builder::new("UnaryBool").imm(boolean));

cranelift-codegen/meta/src/shared/immediates.rs

@@ -29,6 +29,16 @@ pub fn define() -> Vec<OperandKind> {
         .build();
     kinds.push(uimm32);
 
+    // An unsigned 128-bit immediate integer operand.
+    //
+    // This operand is used to pass entire 128-bit vectors as immediates to
+    // instructions like const.
+    let uimm128 = Builder::new_imm("uimm128")
+        .doc("A 128-bit immediate unsigned integer.")
+        .rust_type("ir::Constant")
+        .build();
+    kinds.push(uimm128);
+
     // A 32-bit immediate signed offset.
     //
     // This is used to represent an immediate address offset in load/store

cranelift-codegen/meta/src/shared/instructions.rs

@@ -30,6 +30,7 @@ pub fn define(
     let uimm8 = immediates.by_name("uimm8");
     let uimm32 = immediates.by_name("uimm32");
     let imm64 = immediates.by_name("imm64");
+    let uimm128 = immediates.by_name("uimm128");
     let offset32 = immediates.by_name("offset32");
     let memflags = immediates.by_name("memflags");
     let ieee32 = immediates.by_name("ieee32");
@@ -1088,6 +1089,22 @@ pub fn define(
         .operands_out(vec![a]),
     );
 
+    let N = &operand_doc("N", uimm128, "The 16 immediate bytes of a 128-bit vector");
+    let a = &operand_doc("a", TxN, "A constant vector value");
+
+    ig.push(
+        Inst::new(
+            "vconst",
+            r#"
+        SIMD vector constant.
+
+        Construct a vector with the given immediate bytes.
+        "#,
+        )
+        .operands_in(vec![N])
+        .operands_out(vec![a]),
+    );
+
     let a = &operand_doc("a", Ref, "A constant reference null value");
 
     ig.push(

cranelift-codegen/src/binemit/memorysink.rs

@@ -16,7 +16,7 @@
 use super::{Addend, CodeInfo, CodeOffset, CodeSink, Reloc};
 use crate::binemit::stackmap::Stackmap;
 use crate::ir::entities::Value;
-use crate::ir::{ExternalName, Function, JumpTable, SourceLoc, TrapCode};
+use crate::ir::{ConstantOffset, ExternalName, Function, JumpTable, SourceLoc, TrapCode};
 use crate::isa::TargetIsa;
 use core::ptr::write_unaligned;
 
@@ -78,6 +78,9 @@ pub trait RelocSink {
     /// Add a relocation referencing an external symbol at the current offset.
     fn reloc_external(&mut self, _: CodeOffset, _: Reloc, _: &ExternalName, _: Addend);
 
+    /// Add a relocation referencing a constant.
+    fn reloc_constant(&mut self, _: CodeOffset, _: Reloc, _: ConstantOffset);
+
     /// Add a relocation referencing a jump table.
     fn reloc_jt(&mut self, _: CodeOffset, _: Reloc, _: JumpTable);
 }
@@ -132,6 +135,11 @@ impl<'a> CodeSink for MemoryCodeSink<'a> {
         self.relocs.reloc_external(ofs, rel, name, addend);
     }
 
+    fn reloc_constant(&mut self, rel: Reloc, constant_offset: ConstantOffset) {
+        let ofs = self.offset();
+        self.relocs.reloc_constant(ofs, rel, constant_offset);
+    }
+
     fn reloc_jt(&mut self, rel: Reloc, jt: JumpTable) {
         let ofs = self.offset();
         self.relocs.reloc_jt(ofs, rel, jt);
@@ -169,6 +177,7 @@ pub struct NullRelocSink {}
 impl RelocSink for NullRelocSink {
     fn reloc_ebb(&mut self, _: u32, _: Reloc, _: u32) {}
     fn reloc_external(&mut self, _: u32, _: Reloc, _: &ExternalName, _: i64) {}
+    fn reloc_constant(&mut self, _: CodeOffset, _: Reloc, _: ConstantOffset) {}
     fn reloc_jt(&mut self, _: u32, _: Reloc, _: JumpTable) {}
 }
 

cranelift-codegen/src/binemit/mod.rs

@@ -16,7 +16,7 @@ pub use self::relaxation::relax_branches;
 pub use self::shrink::shrink_instructions;
 pub use self::stackmap::Stackmap;
 use crate::ir::entities::Value;
-use crate::ir::{ExternalName, Function, Inst, JumpTable, SourceLoc, TrapCode};
+use crate::ir::{ConstantOffset, ExternalName, Function, Inst, JumpTable, SourceLoc, TrapCode};
 use crate::isa::TargetIsa;
 pub use crate::regalloc::RegDiversions;
 use core::fmt;
@@ -133,6 +133,9 @@ pub trait CodeSink {
     /// Add a relocation referencing an external symbol plus the addend at the current offset.
     fn reloc_external(&mut self, _: Reloc, _: &ExternalName, _: Addend);
 
+    /// Add a relocation referencing a jump table.
+    fn reloc_constant(&mut self, _: Reloc, _: ConstantOffset);
+
     /// Add a relocation referencing a jump table.
     fn reloc_jt(&mut self, _: Reloc, _: JumpTable);
 
@@ -192,7 +195,13 @@ where
     }
 
     sink.begin_rodata();
-    // TODO: No read-only data (constant pools) at this time.
+
+    // output constants
+    for (_, constant_data) in func.dfg.constants.iter() {
+        for byte in constant_data.iter() {
+            sink.put1(*byte)
+        }
+    }
 
     sink.end_codegen();
 }

cranelift-codegen/src/binemit/relaxation.rs

@@ -37,6 +37,7 @@ use crate::iterators::IteratorExtras;
 use crate::regalloc::RegDiversions;
 use crate::timing;
 use crate::CodegenResult;
+use core::convert::TryFrom;
 use log::debug;
 
 #[cfg(feature = "basic-blocks")]
@@ -135,7 +136,11 @@ pub fn relax_branches(
     let jumptables_size = offset - jumptables;
     let rodata = offset;
 
-    // TODO: Once we have constant pools we'll do some processing here to update offset.
+    for constant in func.dfg.constants.entries_mut() {
+        constant.set_offset(offset);
+        offset +=
+            u32::try_from(constant.len()).expect("Constants must have a length that fits in a u32")
+    }
 
     let rodata_size = offset - rodata;