|
| 1 | +// This file is a part of Julia. License is MIT: https://julialang.org/license |
| 2 | + |
| 3 | +#include "llvm-version.h" |
| 4 | +#include "passes.h" |
| 5 | + |
| 6 | +#include <llvm-c/Core.h> |
| 7 | +#include <llvm-c/Types.h> |
| 8 | + |
| 9 | +#include <llvm/ADT/Statistic.h> |
| 10 | +#include <llvm/Analysis/OptimizationRemarkEmitter.h> |
| 11 | +#include <llvm/IR/Value.h> |
| 12 | +#include <llvm/IR/PassManager.h> |
| 13 | +#include <llvm/IR/Function.h> |
| 14 | +#include <llvm/IR/Instructions.h> |
| 15 | +#include <llvm/IR/IntrinsicInst.h> |
| 16 | +#include <llvm/IR/Module.h> |
| 17 | +#include <llvm/IR/Operator.h> |
| 18 | +#include <llvm/IR/IRBuilder.h> |
| 19 | +#include <llvm/IR/Verifier.h> |
| 20 | +#include <llvm/Pass.h> |
| 21 | +#include <llvm/Support/Debug.h> |
| 22 | + |
| 23 | +#include "julia.h" |
| 24 | +#include "julia_assert.h" |
| 25 | + |
| 26 | +#define DEBUG_TYPE "combine-muladd" |
| 27 | +#undef DEBUG |
| 28 | + |
| 29 | +using namespace llvm; |
| 30 | +STATISTIC(TotalContracted, "Total number of multiplies marked for FMA"); |
| 31 | + |
| 32 | +#ifndef __clang_gcanalyzer__ |
| 33 | +#define REMARK(remark) ORE.emit(remark) |
| 34 | +#else |
| 35 | +#define REMARK(remark) (void) 0; |
| 36 | +#endif |
| 37 | + |
| 38 | +/** |
| 39 | + * Combine |
| 40 | + * ``` |
| 41 | + * %v0 = fmul ... %a, %b |
| 42 | + * %v = fadd contract ... %v0, %c |
| 43 | + * ``` |
| 44 | + * to |
| 45 | + * `%v = call contract @llvm.fmuladd.<...>(... %a, ... %b, ... %c)` |
| 46 | + * when `%v0` has no other use |
| 47 | + */ |
| 48 | + |
| 49 | +// Return true if we changed the mulOp |
| 50 | +static bool checkCombine(Value *maybeMul, OptimizationRemarkEmitter &ORE) JL_NOTSAFEPOINT |
| 51 | +{ |
| 52 | + auto mulOp = dyn_cast<Instruction>(maybeMul); |
| 53 | + if (!mulOp || mulOp->getOpcode() != Instruction::FMul) |
| 54 | + return false; |
| 55 | + if (!mulOp->hasOneUse()) { |
| 56 | + LLVM_DEBUG(dbgs() << "mulOp has multiple uses: " << *maybeMul << "\n"); |
| 57 | + REMARK([&](){ |
| 58 | + return OptimizationRemarkMissed(DEBUG_TYPE, "Multiuse FMul", mulOp) |
| 59 | + << "fmul had multiple uses " << ore::NV("fmul", mulOp); |
| 60 | + }); |
| 61 | + return false; |
| 62 | + } |
| 63 | + // On 5.0+ we only need to mark the mulOp as contract and the backend will do the work for us. |
| 64 | + auto fmf = mulOp->getFastMathFlags(); |
| 65 | + if (!fmf.allowContract()) { |
| 66 | + LLVM_DEBUG(dbgs() << "Marking mulOp for FMA: " << *maybeMul << "\n"); |
| 67 | + REMARK([&](){ |
| 68 | + return OptimizationRemark(DEBUG_TYPE, "Marked for FMA", mulOp) |
| 69 | + << "marked for fma " << ore::NV("fmul", mulOp); |
| 70 | + }); |
| 71 | + ++TotalContracted; |
| 72 | + fmf.setAllowContract(true); |
| 73 | + mulOp->copyFastMathFlags(fmf); |
| 74 | + return true; |
| 75 | + } |
| 76 | + return false; |
| 77 | +} |
| 78 | + |
| 79 | +static bool combineMulAdd(Function &F) JL_NOTSAFEPOINT |
| 80 | +{ |
| 81 | + OptimizationRemarkEmitter ORE(&F); |
| 82 | + bool modified = false; |
| 83 | + for (auto &BB: F) { |
| 84 | + for (auto it = BB.begin(); it != BB.end();) { |
| 85 | + auto &I = *it; |
| 86 | + it++; |
| 87 | + switch (I.getOpcode()) { |
| 88 | + case Instruction::FAdd: { |
| 89 | + if (!I.hasAllowContract()) |
| 90 | + continue; |
| 91 | + modified |= checkCombine(I.getOperand(0), ORE) || checkCombine(I.getOperand(1), ORE); |
| 92 | + break; |
| 93 | + } |
| 94 | + case Instruction::FSub: { |
| 95 | + if (!I.hasAllowContract()) |
| 96 | + continue; |
| 97 | + modified |= checkCombine(I.getOperand(0), ORE) || checkCombine(I.getOperand(1), ORE); |
| 98 | + break; |
| 99 | + } |
| 100 | + default: |
| 101 | + break; |
| 102 | + } |
| 103 | + } |
| 104 | + } |
| 105 | +#ifdef JL_VERIFY_PASSES |
| 106 | + assert(!verifyLLVMIR(F)); |
| 107 | +#endif |
| 108 | + return modified; |
| 109 | +} |
| 110 | + |
| 111 | +PreservedAnalyses CombineMulAddPass::run(Function &F, FunctionAnalysisManager &AM) JL_NOTSAFEPOINT |
| 112 | +{ |
| 113 | + if (combineMulAdd(F)) { |
| 114 | + return PreservedAnalyses::allInSet<CFGAnalyses>(); |
| 115 | + } |
| 116 | + return PreservedAnalyses::all(); |
| 117 | +} |
0 commit comments