Refactor final-gc-lowering (#50741)

Author: pchintalapudi

src/jl_exported_funcs.inc

@@ -583,7 +583,6 @@
     YY(LLVMExtraMPMAddCPUFeaturesPass) \
     YY(LLVMExtraMPMAddRemoveNIPass) \
     YY(LLVMExtraMPMAddLowerSIMDLoopPass) \
-    YY(LLVMExtraMPMAddFinalLowerGCPass) \
     YY(LLVMExtraMPMAddMultiVersioningPass) \
     YY(LLVMExtraMPMAddRemoveJuliaAddrspacesPass) \
     YY(LLVMExtraMPMAddRemoveAddrspacesPass) \
@@ -595,6 +594,7 @@
     YY(LLVMExtraFPMAddPropagateJuliaAddrspacesPass) \
     YY(LLVMExtraFPMAddLowerExcHandlersPass) \
     YY(LLVMExtraFPMAddGCInvariantVerifierPass) \
+    YY(LLVMExtraFPMAddFinalLowerGCPass) \
     YY(LLVMExtraLPMAddJuliaLICMPass) \
     YY(JLJITGetLLVMOrcExecutionSession) \
     YY(JLJITGetJuliaOJIT) \

src/llvm-final-gc-lowering.cpp

@@ -41,8 +41,6 @@ using namespace llvm;
 
 struct FinalLowerGC: private JuliaPassContext {
     bool runOnFunction(Function &F);
-    bool doInitialization(Module &M);
-    bool doFinalization(Module &M);
 
 private:
     Function *queueRootFunc;
@@ -53,7 +51,7 @@ struct FinalLowerGC: private JuliaPassContext {
     Type *T_size;
 
     // Lowers a `julia.new_gc_frame` intrinsic.
-    Value *lowerNewGCFrame(CallInst *target, Function &F);
+    void lowerNewGCFrame(CallInst *target, Function &F);
 
     // Lowers a `julia.push_gc_frame` intrinsic.
     void lowerPushGCFrame(CallInst *target, Function &F);
@@ -62,26 +60,26 @@ struct FinalLowerGC: private JuliaPassContext {
     void lowerPopGCFrame(CallInst *target, Function &F);
 
     // Lowers a `julia.get_gc_frame_slot` intrinsic.
-    Value *lowerGetGCFrameSlot(CallInst *target, Function &F);
+    void lowerGetGCFrameSlot(CallInst *target, Function &F);
 
     // Lowers a `julia.gc_alloc_bytes` intrinsic.
-    Value *lowerGCAllocBytes(CallInst *target, Function &F);
+    void lowerGCAllocBytes(CallInst *target, Function &F);
 
     // Lowers a `julia.queue_gc_root` intrinsic.
-    Value *lowerQueueGCRoot(CallInst *target, Function &F);
+    void lowerQueueGCRoot(CallInst *target, Function &F);
 
     // Lowers a `julia.safepoint` intrinsic.
-    Value *lowerSafepoint(CallInst *target, Function &F);
+    void lowerSafepoint(CallInst *target, Function &F);
 };
 
-Value *FinalLowerGC::lowerNewGCFrame(CallInst *target, Function &F)
+void FinalLowerGC::lowerNewGCFrame(CallInst *target, Function &F)
 {
     ++NewGCFrameCount;
     assert(target->arg_size() == 1);
     unsigned nRoots = cast<ConstantInt>(target->getArgOperand(0))->getLimitedValue(INT_MAX);
 
     // Create the GC frame.
-    IRBuilder<> builder(target->getNextNode());
+    IRBuilder<> builder(target);
     auto gcframe_alloca = builder.CreateAlloca(T_prjlvalue, ConstantInt::get(Type::getInt32Ty(F.getContext()), nRoots + 2));
     gcframe_alloca->setAlignment(Align(16));
     // addrspacecast as needed for non-0 alloca addrspace
@@ -92,7 +90,8 @@ Value *FinalLowerGC::lowerNewGCFrame(CallInst *target, Function &F)
     auto ptrsize = F.getParent()->getDataLayout().getPointerSize();
     builder.CreateMemSet(gcframe, Constant::getNullValue(Type::getInt8Ty(F.getContext())), ptrsize * (nRoots + 2), Align(16), tbaa_gcframe);
 
-    return gcframe;
+    target->replaceAllUsesWith(gcframe);
+    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F)
@@ -102,8 +101,7 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F)
     auto gcframe = target->getArgOperand(0);
     unsigned nRoots = cast<ConstantInt>(target->getArgOperand(1))->getLimitedValue(INT_MAX);
 
-    IRBuilder<> builder(target->getContext());
-    builder.SetInsertPoint(&*(++BasicBlock::iterator(target)));
+    IRBuilder<> builder(target);
     StoreInst *inst = builder.CreateAlignedStore(
                 ConstantInt::get(T_size, JL_GC_ENCODE_PUSHARGS(nRoots)),
                 builder.CreateBitCast(
@@ -123,6 +121,7 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F)
             gcframe,
             builder.CreateBitCast(pgcstack, PointerType::get(PointerType::get(T_prjlvalue, 0), 0)),
             Align(sizeof(void*)));
+    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerPopGCFrame(CallInst *target, Function &F)
@@ -131,8 +130,7 @@ void FinalLowerGC::lowerPopGCFrame(CallInst *target, Function &F)
     assert(target->arg_size() == 1);
     auto gcframe = target->getArgOperand(0);
 
-    IRBuilder<> builder(target->getContext());
-    builder.SetInsertPoint(target);
+    IRBuilder<> builder(target);
     Instruction *gcpop =
         cast<Instruction>(builder.CreateConstInBoundsGEP1_32(T_prjlvalue, gcframe, 1));
     Instruction *inst = builder.CreateAlignedLoad(T_prjlvalue, gcpop, Align(sizeof(void*)), "frame.prev");
@@ -143,55 +141,53 @@ void FinalLowerGC::lowerPopGCFrame(CallInst *target, Function &F)
             PointerType::get(T_prjlvalue, 0)),
         Align(sizeof(void*)));
     inst->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe);
+    target->eraseFromParent();
 }
 
-Value *FinalLowerGC::lowerGetGCFrameSlot(CallInst *target, Function &F)
+void FinalLowerGC::lowerGetGCFrameSlot(CallInst *target, Function &F)
 {
     ++GetGCFrameSlotCount;
     assert(target->arg_size() == 2);
     auto gcframe = target->getArgOperand(0);
     auto index = target->getArgOperand(1);
 
     // Initialize an IR builder.
-    IRBuilder<> builder(target->getContext());
-    builder.SetInsertPoint(target);
+    IRBuilder<> builder(target);
 
     // The first two slots are reserved, so we'll add two to the index.
     index = builder.CreateAdd(index, ConstantInt::get(Type::getInt32Ty(F.getContext()), 2));
 
     // Lower the intrinsic as a GEP.
     auto gep = builder.CreateInBoundsGEP(T_prjlvalue, gcframe, index);
     gep->takeName(target);
-    return gep;
+    target->replaceAllUsesWith(gep);
+    target->eraseFromParent();
 }
 
-Value *FinalLowerGC::lowerQueueGCRoot(CallInst *target, Function &F)
+void FinalLowerGC::lowerQueueGCRoot(CallInst *target, Function &F)
 {
     ++QueueGCRootCount;
     assert(target->arg_size() == 1);
     target->setCalledFunction(queueRootFunc);
-    return target;
 }
 
-Value *FinalLowerGC::lowerSafepoint(CallInst *target, Function &F)
+void FinalLowerGC::lowerSafepoint(CallInst *target, Function &F)
 {
     ++SafepointCount;
     assert(target->arg_size() == 1);
-    IRBuilder<> builder(target->getContext());
-    builder.SetInsertPoint(target);
+    IRBuilder<> builder(target);
     Value* signal_page = target->getOperand(0);
-    Value* load = builder.CreateLoad(T_size, signal_page, true);
-    return load;
+    builder.CreateLoad(T_size, signal_page, true);
+    target->eraseFromParent();
 }
 
-Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
+void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
 {
     ++GCAllocBytesCount;
     assert(target->arg_size() == 3);
     CallInst *newI;
 
     IRBuilder<> builder(target);
-    builder.SetCurrentDebugLocation(target->getDebugLoc());
     auto ptls = target->getArgOperand(0);
     auto type = target->getArgOperand(2);
     Attribute derefAttr;
@@ -222,86 +218,13 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
     newI->setAttributes(newI->getCalledFunction()->getAttributes());
     newI->addRetAttr(derefAttr);
     newI->takeName(target);
-    return newI;
-}
-
-bool FinalLowerGC::doInitialization(Module &M) {
-    // Initialize platform-agnostic references.
-    initAll(M);
-
-    // Initialize platform-specific references.
-    queueRootFunc = getOrDeclare(jl_well_known::GCQueueRoot);
-    poolAllocFunc = getOrDeclare(jl_well_known::GCPoolAlloc);
-    bigAllocFunc = getOrDeclare(jl_well_known::GCBigAlloc);
-    allocTypedFunc = getOrDeclare(jl_well_known::GCAllocTyped);
-    T_size = M.getDataLayout().getIntPtrType(M.getContext());
-
-    GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, allocTypedFunc};
-    unsigned j = 0;
-    for (unsigned i = 0; i < sizeof(functionList) / sizeof(void*); i++) {
-        if (!functionList[i])
-            continue;
-        if (i != j)
-            functionList[j] = functionList[i];
-        j++;
-    }
-    if (j != 0)
-        appendToCompilerUsed(M, ArrayRef<GlobalValue*>(functionList, j));
-    return true;
-}
-
-bool FinalLowerGC::doFinalization(Module &M)
-{
-    GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, allocTypedFunc};
-    queueRootFunc = poolAllocFunc = bigAllocFunc = allocTypedFunc = nullptr;
-    auto used = M.getGlobalVariable("llvm.compiler.used");
-    if (!used)
-        return false;
-    SmallPtrSet<Constant*, 16> InitAsSet(
-        functionList,
-        functionList + sizeof(functionList) / sizeof(void*));
-    bool changed = false;
-    SmallVector<Constant*, 16> init;
-    ConstantArray *CA = cast<ConstantArray>(used->getInitializer());
-    for (auto &Op : CA->operands()) {
-        Constant *C = cast_or_null<Constant>(Op);
-        if (InitAsSet.count(C->stripPointerCasts())) {
-            changed = true;
-            continue;
-        }
-        init.push_back(C);
-    }
-    if (!changed)
-        return false;
-    used->eraseFromParent();
-    if (init.empty())
-        return true;
-    ArrayType *ATy = ArrayType::get(Type::getInt8PtrTy(M.getContext()), init.size());
-    used = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
-                                    ConstantArray::get(ATy, init), "llvm.compiler.used");
-    used->setSection("llvm.metadata");
-    return true;
-}
-
-template<typename TIterator>
-static void replaceInstruction(
-    Instruction *oldInstruction,
-    Value *newInstruction,
-    TIterator &it)
-{
-    if (newInstruction != oldInstruction) {
-        oldInstruction->replaceAllUsesWith(newInstruction);
-        it = oldInstruction->eraseFromParent();
-    }
-    else {
-        ++it;
-    }
+    target->replaceAllUsesWith(newI);
+    target->eraseFromParent();
 }
 
 bool FinalLowerGC::runOnFunction(Function &F)
 {
-    // Check availability of functions again since they might have been deleted.
-    initFunctions(*F.getParent());
+    initAll(*F.getParent());
     if (!pgcstack_getter && !adoptthread_func) {
         LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Skipping function " << F.getName() << "\n");
         return false;
@@ -314,55 +237,39 @@ bool FinalLowerGC::runOnFunction(Function &F)
         return false;
     }
     LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Processing function " << F.getName() << "\n");
-
-    // Acquire intrinsic functions.
-    auto newGCFrameFunc = getOrNull(jl_intrinsics::newGCFrame);
-    auto pushGCFrameFunc = getOrNull(jl_intrinsics::pushGCFrame);
-    auto popGCFrameFunc = getOrNull(jl_intrinsics::popGCFrame);
-    auto getGCFrameSlotFunc = getOrNull(jl_intrinsics::getGCFrameSlot);
-    auto GCAllocBytesFunc = getOrNull(jl_intrinsics::GCAllocBytes);
-    auto queueGCRootFunc = getOrNull(jl_intrinsics::queueGCRoot);
-    auto safepointFunc = getOrNull(jl_intrinsics::safepoint);
+    queueRootFunc = getOrDeclare(jl_well_known::GCQueueRoot);
+    poolAllocFunc = getOrDeclare(jl_well_known::GCPoolAlloc);
+    bigAllocFunc = getOrDeclare(jl_well_known::GCBigAlloc);
+    allocTypedFunc = getOrDeclare(jl_well_known::GCAllocTyped);
+    T_size = F.getParent()->getDataLayout().getIntPtrType(F.getContext());
 
     // Lower all calls to supported intrinsics.
-    for (BasicBlock &BB : F) {
-        for (auto it = BB.begin(); it != BB.end();) {
-            auto *CI = dyn_cast<CallInst>(&*it);
-            if (!CI) {
-                ++it;
+    for (auto &BB : F) {
+        for (auto &I : make_early_inc_range(BB)) {
+            auto *CI = dyn_cast<CallInst>(&I);
+            if (!CI)
                 continue;
-            }
 
             Value *callee = CI->getCalledOperand();
             assert(callee);
 
-            if (callee == newGCFrameFunc) {
-                replaceInstruction(CI, lowerNewGCFrame(CI, F), it);
-            }
-            else if (callee == pushGCFrameFunc) {
-                lowerPushGCFrame(CI, F);
-                it = CI->eraseFromParent();
-            }
-            else if (callee == popGCFrameFunc) {
-                lowerPopGCFrame(CI, F);
-                it = CI->eraseFromParent();
-            }
-            else if (callee == getGCFrameSlotFunc) {
-                replaceInstruction(CI, lowerGetGCFrameSlot(CI, F), it);
-            }
-            else if (callee == GCAllocBytesFunc) {
-                replaceInstruction(CI, lowerGCAllocBytes(CI, F), it);
-            }
-            else if (callee == queueGCRootFunc) {
-                replaceInstruction(CI, lowerQueueGCRoot(CI, F), it);
-            }
-            else if (callee == safepointFunc) {
-                lowerSafepoint(CI, F);
-                it = CI->eraseFromParent();
-            }
-            else {
-                ++it;
-            }
+#define LOWER_INTRINSIC(INTRINSIC, LOWER_INTRINSIC_FUNC) \
+            do { \
+                auto intrinsic = getOrNull(jl_intrinsics::INTRINSIC); \
+                if (intrinsic == callee) { \
+                    LOWER_INTRINSIC_FUNC(CI, F); \
+                } \
+            } while (0)
+
+            LOWER_INTRINSIC(newGCFrame, lowerNewGCFrame);
+            LOWER_INTRINSIC(pushGCFrame, lowerPushGCFrame);
+            LOWER_INTRINSIC(popGCFrame, lowerPopGCFrame);
+            LOWER_INTRINSIC(getGCFrameSlot, lowerGetGCFrameSlot);
+            LOWER_INTRINSIC(GCAllocBytes, lowerGCAllocBytes);
+            LOWER_INTRINSIC(queueGCRoot, lowerQueueGCRoot);
+            LOWER_INTRINSIC(safepoint, lowerSafepoint);
+
+#undef LOWER_INTRINSIC
         }
     }
 
@@ -380,44 +287,24 @@ struct FinalLowerGCLegacy: public FunctionPass {
 
 private:
     bool runOnFunction(Function &F) override;
-    bool doInitialization(Module &M) override;
-    bool doFinalization(Module &M) override;
 
     FinalLowerGC finalLowerGC;
 };
 
 bool FinalLowerGCLegacy::runOnFunction(Function &F) {
-    return finalLowerGC.runOnFunction(F);
-}
-
-bool FinalLowerGCLegacy::doInitialization(Module &M) {
-    return finalLowerGC.doInitialization(M);
-}
-
-bool FinalLowerGCLegacy::doFinalization(Module &M) {
-    auto ret = finalLowerGC.doFinalization(M);
+    auto modified = finalLowerGC.runOnFunction(F);
 #ifdef JL_VERIFY_PASSES
-    assert(!verifyLLVMIR(M));
+    assert(!verifyLLVMIR(F));
 #endif
-    return ret;
+    return modified;
 }
 
-
-PreservedAnalyses FinalLowerGCPass::run(Module &M, ModuleAnalysisManager &AM)
+PreservedAnalyses FinalLowerGCPass::run(Function &F, FunctionAnalysisManager &AM)
 {
-    auto finalLowerGC = FinalLowerGC();
-    bool modified = false;
-    modified |= finalLowerGC.doInitialization(M);
-    for (auto &F : M.functions()) {
-        if (F.isDeclaration())
-            continue;
-        modified |= finalLowerGC.runOnFunction(F);
-    }
-    modified |= finalLowerGC.doFinalization(M);
+    if (FinalLowerGC().runOnFunction(F)) {
 #ifdef JL_VERIFY_PASSES
-    assert(!verifyLLVMIR(M));
+        assert(!verifyLLVMIR(F));
 #endif
-    if (modified) {
         return PreservedAnalyses::allInSet<CFGAnalyses>();
     }
     return PreservedAnalyses::all();