Add support for split barriers extension SPV_INTEL_split_barrier (#1424)

bashbaug · web-flow · commit e3cd29689a21 · 2022-08-31T19:13:24.000+02:00
This PR adds support for split barriers and the SPV_INTEL_split_barrier extension. The related SPIR-V extension spec can be found here: * KhronosGroup/SPIRV-Registry#136 The related OpenCL C extension spec can be found here: * KhronosGroup/OpenCL-Docs#765
diff --git a/include/LLVMSPIRVExtensions.inc b/include/LLVMSPIRVExtensions.inc
@@ -53,3 +53,4 @@ EXT(SPV_INTEL_hw_thread_queries)
 EXT(SPV_INTEL_global_variable_decorations)
 EXT(SPV_INTEL_non_constant_addrspace_printf)
 EXT(SPV_INTEL_complex_float_mul_div)
+EXT(SPV_INTEL_split_barrier)
diff --git a/lib/SPIRV/OCLToSPIRV.cpp b/lib/SPIRV/OCLToSPIRV.cpp
@@ -379,6 +379,10 @@ void OCLToSPIRVBase::visitCallInst(CallInst &CI) {
     visitSubgroupImageMediaBlockINTEL(&CI, DemangledName);
     return;
   }
+  if (DemangledName.find(kOCLBuiltinName::SplitBarrierINTELPrefix) == 0) {
+    visitCallSplitBarrierINTEL(&CI, DemangledName);
+    return;
+  }
   // Handle 'cl_intel_device_side_avc_motion_estimation' extension built-ins
   if (DemangledName.find(kOCLSubgroupsAVCIntel::Prefix) == 0 ||
       // Workaround for a bug in the extension specification
@@ -1870,6 +1874,37 @@ void OCLToSPIRVBase::visitSubgroupAVCBuiltinCallWithSampler(
       &Attrs);
 }
 
+void OCLToSPIRVBase::visitCallSplitBarrierINTEL(CallInst *CI,
+                                                StringRef DemangledName) {
+  auto Lit = getBarrierLiterals(CI);
+  AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+  Op OpCode =
+      StringSwitch<Op>(DemangledName)
+          .Case("intel_work_group_barrier_arrive", OpControlBarrierArriveINTEL)
+          .Case("intel_work_group_barrier_wait", OpControlBarrierWaitINTEL)
+          .Default(OpNop);
+
+  mutateCallInstSPIRV(
+      M, CI,
+      [=](CallInst *, std::vector<Value *> &Args) {
+        Args.resize(3);
+        // Execution scope
+        Args[0] = addInt32(map<Scope>(std::get<2>(Lit)));
+        // Memory scope
+        Args[1] = addInt32(map<Scope>(std::get<1>(Lit)));
+        // Memory semantics
+        // OpControlBarrierArriveINTEL -> Release,
+        // OpControlBarrierWaitINTEL -> Acquire
+        unsigned MemFenceFlag = std::get<0>(Lit);
+        OCLMemOrderKind MemOrder = OpCode == OpControlBarrierArriveINTEL
+                                       ? OCLMO_release
+                                       : OCLMO_acquire;
+        Args[2] = addInt32(mapOCLMemSemanticToSPIRV(MemFenceFlag, MemOrder));
+        return getSPIRVFuncName(OpCode);
+      },
+      &Attrs);
+}
+
 void OCLToSPIRVBase::visitCallLdexp(CallInst *CI, StringRef MangledName,
                                     StringRef DemangledName) {
   auto Args = getArguments(CI);
diff --git a/lib/SPIRV/OCLToSPIRV.h b/lib/SPIRV/OCLToSPIRV.h
@@ -247,6 +247,9 @@ class OCLToSPIRVBase : public InstVisitor<OCLToSPIRVBase> {
   void visitSubgroupAVCBuiltinCallWithSampler(CallInst *CI,
                                               StringRef DemangledName);
 
+  /// For cl_intel_split_work_group_barrier built-ins:
+  void visitCallSplitBarrierINTEL(CallInst *CI, StringRef DemangledName);
+
   void visitCallLdexp(CallInst *CI, StringRef MangledName,
                       StringRef DemangledName);
 
diff --git a/lib/SPIRV/OCLUtil.cpp b/lib/SPIRV/OCLUtil.cpp
@@ -432,6 +432,9 @@ template <> void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
   _SPIRV_OP(bitfield_extract_signed, BitFieldSExtract)
   _SPIRV_OP(bitfield_extract_unsigned, BitFieldUExtract)
   _SPIRV_OP(bit_reverse, BitReverse)
+  // cl_khr_split_work_group_barrier
+  _SPIRV_OP(intel_work_group_barrier_arrive, ControlBarrierArriveINTEL)
+  _SPIRV_OP(intel_work_group_barrier_wait, ControlBarrierWaitINTEL)
 #undef _SPIRV_OP
 }
 
@@ -1038,7 +1041,9 @@ class OCLBuiltinFuncMangleInfo : public SPIRV::BuiltinFuncMangleInfo {
     } else if (NameRef.contains("barrier")) {
       addUnsignedArg(0);
       if (NameRef.equals("work_group_barrier") ||
-          NameRef.equals("sub_group_barrier"))
+          NameRef.equals("sub_group_barrier") ||
+          NameRef.equals("intel_work_group_barrier_arrive") ||
+          NameRef.equals("intel_work_group_barrier_wait"))
         setEnumArg(1, SPIR::PRIMITIVE_MEMORY_SCOPE);
     } else if (NameRef.startswith("atomic_work_item_fence")) {
       addUnsignedArg(0);
diff --git a/lib/SPIRV/OCLUtil.h b/lib/SPIRV/OCLUtil.h
@@ -305,6 +305,7 @@ const static char SubgroupBlockWriteINTELPrefix[] =
     "intel_sub_group_block_write";
 const static char SubgroupImageMediaBlockINTELPrefix[] =
     "intel_sub_group_media_block";
+const static char SplitBarrierINTELPrefix[] = "intel_work_group_barrier_";
 const static char LDEXP[] = "ldexp";
 #define _SPIRV_OP(x)                                                           \
   const static char ConvertBFloat16##x##AsUShort##x[] =                        \
diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp
@@ -3099,7 +3099,7 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName,
       Func->addFnAttr(Attribute::NoUnwind);
     auto OC = BI->getOpCode();
     if (isGroupOpCode(OC) || isIntelSubgroupOpCode(OC) ||
-        OC == OpControlBarrier)
+        isSplitBarrierINTELOpCode(OC) || OC == OpControlBarrier)
       Func->addFnAttr(Attribute::Convergent);
   }
   auto Call =
diff --git a/lib/SPIRV/SPIRVToOCL.cpp b/lib/SPIRV/SPIRVToOCL.cpp
@@ -119,6 +119,10 @@ void SPIRVToOCLBase::visitCallInst(CallInst &CI) {
   if (OC == OpControlBarrier) {
     visitCallSPIRVControlBarrier(&CI);
   }
+  if (isSplitBarrierINTELOpCode(OC)) {
+    visitCallSPIRVSplitBarrierINTEL(&CI, OC);
+    return;
+  }
   if (isAtomicOpCode(OC)) {
     visitCallSPIRVAtomicBuiltin(&CI, OC);
     return;
diff --git a/lib/SPIRV/SPIRVToOCL.h b/lib/SPIRV/SPIRVToOCL.h
@@ -222,6 +222,12 @@ class SPIRVToOCLBase : public InstVisitor<SPIRVToOCLBase> {
   /// - OCL1.2: barrier
   virtual void visitCallSPIRVControlBarrier(CallInst *CI) = 0;
 
+  /// Transform split __spirv_ControlBarrierArriveINTEL and
+  /// __spirv_ControlBarrierWaitINTEL barrier to:
+  /// - OCL2.0: overload with a memory_scope argument
+  /// - OCL1.2: overload with no memory_scope argument
+  virtual void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) = 0;
+
   /// Transform __spirv_EnqueueKernel to __enqueue_kernel
   virtual void visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) = 0;
 
@@ -305,6 +311,11 @@ class SPIRVToOCL12Base : public SPIRVToOCLBase {
   ///       barrier(flag(sema))
   void visitCallSPIRVControlBarrier(CallInst *CI) override;
 
+  /// Transform split __spirv_ControlBarrierArriveINTEL and
+  /// __spirv_ControlBarrierWaitINTEL barrier to overloads without a
+  /// memory_scope argument.
+  void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) override;
+
   /// Transform __spirv_OpAtomic functions. It firstly conduct generic
   /// mutations for all builtins and then mutate some of them seperately
   Instruction *visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) override;
@@ -394,6 +405,11 @@ class SPIRVToOCL20Base : public SPIRVToOCLBase {
   ///         sub_group_barrier(flag(sema), map(memScope))
   void visitCallSPIRVControlBarrier(CallInst *CI) override;
 
+  /// Transform split __spirv_ControlBarrierArriveINTEL and
+  /// __spirv_ControlBarrierWaitINTEL barrier to overloads with a
+  /// memory_scope argument.
+  void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) override;
+
   /// Transform __spirv_Atomic* to atomic_*.
   ///   __spirv_Atomic*(atomic_op, scope, sema, ops, ...) =>
   ///      atomic_*(generic atomic_op, ops, ..., order(sema), map(scope))
diff --git a/lib/SPIRV/SPIRVToOCL12.cpp b/lib/SPIRV/SPIRVToOCL12.cpp
@@ -100,6 +100,19 @@ void SPIRVToOCL12Base::visitCallSPIRVControlBarrier(CallInst *CI) {
       &Attrs);
 }
 
+void SPIRVToOCL12Base::visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) {
+  AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+  mutateCallInstOCL(
+      M, CI,
+      [=](CallInst *, std::vector<Value *> &Args) {
+        Value *MemFenceFlags =
+            SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI);
+        Args.assign(1, MemFenceFlags);
+        return OCLSPIRVBuiltinMap::rmap(OC);
+      },
+      &Attrs);
+}
+
 Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) {
   AttributeList Attrs = CI->getCalledFunction()->getAttributes();
   return mutateCallInstOCL(
diff --git a/lib/SPIRV/SPIRVToOCL20.cpp b/lib/SPIRV/SPIRVToOCL20.cpp
@@ -123,6 +123,28 @@ void SPIRVToOCL20Base::visitCallSPIRVControlBarrier(CallInst *CI) {
       &NewAttrs);
 }
 
+void SPIRVToOCL20Base::visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) {
+  AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+  mutateCallInstOCL(
+      M, CI,
+      [=](CallInst *, std::vector<Value *> &Args) {
+        auto GetArg = [=](unsigned I) {
+          return cast<ConstantInt>(Args[I])->getZExtValue();
+        };
+        Value *MemScope =
+            getInt32(M, rmap<OCLScopeKind>(static_cast<Scope>(GetArg(1))));
+        Value *MemFenceFlags =
+            SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI);
+
+        Args.resize(2);
+        Args[0] = MemFenceFlags;
+        Args[1] = MemScope;
+
+        return OCLSPIRVBuiltinMap::rmap(OC);
+      },
+      &Attrs);
+}
+
 std::string SPIRVToOCL20Base::mapFPAtomicName(Op OC) {
   assert(isFPAtomicOpCode(OC) && "Not intended to handle other opcodes than "
                                  "AtomicF{Add/Min/Max}EXT!");
diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.h b/lib/SPIRV/libSPIRV/SPIRVInstruction.h
@@ -3327,6 +3327,24 @@ _SPIRV_OP(JointMatrixMad, true, 7)
 _SPIRV_OP(JointMatrixWorkItemLength, true, 4)
 #undef _SPIRV_OP
 
+class SPIRVSplitBarrierINTELBase : public SPIRVInstTemplateBase {
+protected:
+  SPIRVCapVec getRequiredCapability() const override {
+    return getVec(CapabilitySplitBarrierINTEL);
+  }
+
+  llvm::Optional<ExtensionID> getRequiredExtension() const override {
+    return ExtensionID::SPV_INTEL_split_barrier;
+  }
+};
+
+#define _SPIRV_OP(x, ...)                                                      \
+  typedef SPIRVInstTemplate<SPIRVSplitBarrierINTELBase, Op##x, __VA_ARGS__>    \
+      SPIRV##x;
+_SPIRV_OP(ControlBarrierArriveINTEL, false, 4)
+_SPIRV_OP(ControlBarrierWaitINTEL, false, 4)
+#undef _SPIRV_OP
+
 class SPIRVGroupUniformArithmeticKHRInstBase : public SPIRVInstTemplateBase {
 public:
   SPIRVCapVec getRequiredCapability() const override {
diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCode.h b/lib/SPIRV/libSPIRV/SPIRVOpCode.h
@@ -252,6 +252,11 @@ inline bool isEventOpCode(Op OpCode) {
   return OpRetainEvent <= OpCode && OpCode <= OpCaptureEventProfilingInfo;
 }
 
+inline bool isSplitBarrierINTELOpCode(Op OpCode) {
+  return OpCode == OpControlBarrierArriveINTEL ||
+         OpCode == OpControlBarrierWaitINTEL;
+}
+
 } // namespace SPIRV
 
 #endif // SPIRV_LIBSPIRV_SPIRVOPCODE_H
diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h b/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h
@@ -547,6 +547,8 @@ _SPIRV_OP(TypeBufferSurfaceINTEL, 6086)
 _SPIRV_OP(TypeStructContinuedINTEL, 6090)
 _SPIRV_OP(ConstantCompositeContinuedINTEL, 6091)
 _SPIRV_OP(SpecConstantCompositeContinuedINTEL, 6092)
+_SPIRV_OP(ControlBarrierArriveINTEL, 6142)
+_SPIRV_OP(ControlBarrierWaitINTEL, 6143)
 _SPIRV_OP(GroupIMulKHR, 6401)
 _SPIRV_OP(GroupFMulKHR, 6402)
 _SPIRV_OP(GroupBitwiseAndKHR, 6403)
diff --git a/test/transcoding/SPV_INTEL_split_barrier/split_work_group_barrier_12.ll b/test/transcoding/SPV_INTEL_split_barrier/split_work_group_barrier_12.ll
@@ -0,0 +1,99 @@
+;; kernel void test(global uint* dst)
+;; {
+;;     intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE);
+;;     intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE);
+;;     intel_work_group_barrier_arrive(CLK_GLOBAL_MEM_FENCE);
+;;     intel_work_group_barrier_wait(CLK_GLOBAL_MEM_FENCE);
+;;
+;;     intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
+;;     intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
+;;}
+
+; Test for SPV_INTEL_split_barrier (OpenCL C LLVM IR)
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc -o %t.spv --spirv-ext=+SPV_INTEL_split_barrier
+; RUN: llvm-spirv %t.spv -o %t.spt --to-text
+; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv %t.spv -o %t.rev.bc -r --spirv-target-env=CL1.2
+; RUN: llvm-dis %t.rev.bc -o %t.rev.ll
+; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM
+
+; RUN: not llvm-spirv %t.bc 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; CHECK-ERROR: RequiresExtension: Feature requires the following SPIR-V extension:
+; CHECK-ERROR-NEXT: SPV_INTEL_split_barrier
+
+; ModuleID = 'split_barrier.cl'
+source_filename = "split_barrier.cl"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64"
+
+; CHECK-SPIRV: Capability SplitBarrierINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_split_barrier"
+; CHECK-SPIRV: TypeInt [[UINT:[0-9]+]] 32 0
+;
+; Scopes:
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_WORK_GROUP:[0-9]+]] 2
+;
+; Memory Semantics:
+; 0x2 Acquire + 0x100 WorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQUIRE_LOCAL:[0-9]+]] 258
+; 0x4 Release + 0x100 WorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[RELEASE_LOCAL:[0-9]+]] 260
+; 0x2 Acquire + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQUIRE_GLOBAL:[0-9]+]] 514
+; 0x4 Release + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[RELEASE_GLOBAL:[0-9]+]] 516
+; 0x2 Acquire + 0x100 WorkgroupMemory + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQUIRE_LOCAL_GLOBAL:[0-9]+]] 770
+; 0x4 Release + 0x100 WorkgroupMemory + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[RELEASE_LOCAL_GLOBAL:[0-9]+]] 772
+;
+; CHECK-SPIRV: ControlBarrierArriveINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[RELEASE_LOCAL]]
+; CHECK-SPIRV: ControlBarrierWaitINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[ACQUIRE_LOCAL]]
+; CHECK-SPIRV: ControlBarrierArriveINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[RELEASE_GLOBAL]]
+; CHECK-SPIRV: ControlBarrierWaitINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[ACQUIRE_GLOBAL]]
+;
+; CHECK-SPIRV: ControlBarrierArriveINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[RELEASE_LOCAL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrierWaitINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[ACQUIRE_LOCAL_GLOBAL]]
+
+; CHECK-LLVM-LABEL: define spir_kernel void @test
+; Function Attrs: convergent norecurse nounwind
+define dso_local spir_kernel void @test(i32 addrspace(1)* nocapture noundef readnone align 4 %0) local_unnamed_addr #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 {
+  tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 1) #2
+    ; CHECK-LLVM: call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 1)
+  tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 1) #2
+    ; CHECK-LLVM: call spir_func void @_Z29intel_work_group_barrier_waitj(i32 1)
+  tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 2) #2
+    ; CHECK-LLVM: call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 2)
+  tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 2) #2
+    ; CHECK-LLVM: call spir_func void @_Z29intel_work_group_barrier_waitj(i32 2)
+  tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 3) #2
+    ; CHECK-LLVM: call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 3)
+  tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 3) #2
+    ; CHECK-LLVM: call spir_func void @_Z29intel_work_group_barrier_waitj(i32 3)
+  ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef) local_unnamed_addr #1
+
+attributes #0 = { convergent norecurse nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent nounwind }
+
+!llvm.module.flags = !{!0, !1}
+!opencl.ocl.version = !{!2}
+!opencl.spir.version = !{!2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"frame-pointer", i32 2}
+!2 = !{i32 1, i32 2}
+!3 = !{!"clang version 15.0.0 (https://github.com/llvm/llvm-project 861386dbd6ff0d91636b7c674c2abb2eccd9d3f2)"}
+!4 = !{i32 1}
+!5 = !{!"none"}
+!6 = !{!"uint*"}
+!7 = !{!""}
diff --git a/test/transcoding/SPV_INTEL_split_barrier/split_work_group_barrier_20.ll b/test/transcoding/SPV_INTEL_split_barrier/split_work_group_barrier_20.ll
diff --git a/test/transcoding/SPV_INTEL_split_barrier/split_work_group_barrier_spirv.ll b/test/transcoding/SPV_INTEL_split_barrier/split_work_group_barrier_spirv.ll

Original file line number	Diff line number	Diff line change
`@@ -3099,7 +3099,7 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName,`
`3099`	`3099`	`Func->addFnAttr(Attribute::NoUnwind);`
`3100`	`3100`	`auto OC = BI->getOpCode();`
`3101`	`3101`	`if (isGroupOpCode(OC) \|\| isIntelSubgroupOpCode(OC) \|\|`
`3102`		`- OC == OpControlBarrier)`
	`3102`	`+ isSplitBarrierINTELOpCode(OC) \|\| OC == OpControlBarrier)`
`3103`	`3103`	`Func->addFnAttr(Attribute::Convergent);`
`3104`	`3104`	`}`
`3105`	`3105`	`auto Call =`