From: Chandler Carruth Date: Sat, 24 Jan 2015 10:18:47 +0000 (+0000) Subject: [PM] Move the LowerExpectIntrinsic pass to the Scalar library. X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=93647f18afb1ccf1a9b609b5db827b5f422b4d78;p=oota-llvm.git [PM] Move the LowerExpectIntrinsic pass to the Scalar library. It was already in the Scalar header and referenced extensively as being in this library, the source file was just in the utils directory for some reason. No actual functionality changed. I noticed as it didn't make sense to add a pass header to the utils headers. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@226991 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/lib/Transforms/Scalar/CMakeLists.txt b/lib/Transforms/Scalar/CMakeLists.txt index 3511bb16d4e..1279e83d865 100644 --- a/lib/Transforms/Scalar/CMakeLists.txt +++ b/lib/Transforms/Scalar/CMakeLists.txt @@ -23,6 +23,7 @@ add_llvm_library(LLVMScalarOpts LoopUnrollPass.cpp LoopUnswitch.cpp LowerAtomic.cpp + LowerExpectIntrinsic.cpp MemCpyOptimizer.cpp MergedLoadStoreMotion.cpp PartiallyInlineLibCalls.cpp diff --git a/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp new file mode 100644 index 00000000000..ff89e747440 --- /dev/null +++ b/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -0,0 +1,188 @@ +//===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass lowers the 'expect' intrinsic to LLVM metadata. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Scalar.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Metadata.h" +#include "llvm/Pass.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include + +using namespace llvm; + +#define DEBUG_TYPE "lower-expect-intrinsic" + +STATISTIC(IfHandled, "Number of 'expect' intrinsic instructions handled"); + +static cl::opt +LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64), + cl::desc("Weight of the branch likely to be taken (default = 64)")); +static cl::opt +UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4), + cl::desc("Weight of the branch unlikely to be taken (default = 4)")); + +namespace { + + class LowerExpectIntrinsic : public FunctionPass { + + bool HandleSwitchExpect(SwitchInst *SI); + + bool HandleIfExpect(BranchInst *BI); + + public: + static char ID; + LowerExpectIntrinsic() : FunctionPass(ID) { + initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override; + }; +} + + +bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) { + CallInst *CI = dyn_cast(SI->getCondition()); + if (!CI) + return false; + + Function *Fn = CI->getCalledFunction(); + if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) + return false; + + Value *ArgValue = CI->getArgOperand(0); + ConstantInt *ExpectedValue = dyn_cast(CI->getArgOperand(1)); + if (!ExpectedValue) + return false; + + SwitchInst::CaseIt Case = SI->findCaseValue(ExpectedValue); + unsigned n = SI->getNumCases(); // +1 for default case. + std::vector Weights(n + 1); + + Weights[0] = Case == SI->case_default() ? LikelyBranchWeight + : UnlikelyBranchWeight; + for (unsigned i = 0; i != n; ++i) + Weights[i + 1] = i == Case.getCaseIndex() ? LikelyBranchWeight + : UnlikelyBranchWeight; + + SI->setMetadata(LLVMContext::MD_prof, + MDBuilder(CI->getContext()).createBranchWeights(Weights)); + + SI->setCondition(ArgValue); + return true; +} + + +bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) { + if (BI->isUnconditional()) + return false; + + // Handle non-optimized IR code like: + // %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1) + // %tobool = icmp ne i64 %expval, 0 + // br i1 %tobool, label %if.then, label %if.end + // + // Or the following simpler case: + // %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1) + // br i1 %expval, label %if.then, label %if.end + + CallInst *CI; + + ICmpInst *CmpI = dyn_cast(BI->getCondition()); + if (!CmpI) { + CI = dyn_cast(BI->getCondition()); + } else { + if (CmpI->getPredicate() != CmpInst::ICMP_NE) + return false; + CI = dyn_cast(CmpI->getOperand(0)); + } + + if (!CI) + return false; + + Function *Fn = CI->getCalledFunction(); + if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) + return false; + + Value *ArgValue = CI->getArgOperand(0); + ConstantInt *ExpectedValue = dyn_cast(CI->getArgOperand(1)); + if (!ExpectedValue) + return false; + + MDBuilder MDB(CI->getContext()); + MDNode *Node; + + // If expect value is equal to 1 it means that we are more likely to take + // branch 0, in other case more likely is branch 1. + if (ExpectedValue->isOne()) + Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); + else + Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); + + BI->setMetadata(LLVMContext::MD_prof, Node); + + if (CmpI) + CmpI->setOperand(0, ArgValue); + else + BI->setCondition(ArgValue); + return true; +} + + +bool LowerExpectIntrinsic::runOnFunction(Function &F) { + for (Function::iterator I = F.begin(), E = F.end(); I != E;) { + BasicBlock *BB = I++; + + // Create "block_weights" metadata. + if (BranchInst *BI = dyn_cast(BB->getTerminator())) { + if (HandleIfExpect(BI)) + IfHandled++; + } else if (SwitchInst *SI = dyn_cast(BB->getTerminator())) { + if (HandleSwitchExpect(SI)) + IfHandled++; + } + + // remove llvm.expect intrinsics. + for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); + BI != BE; ) { + CallInst *CI = dyn_cast(BI++); + if (!CI) + continue; + + Function *Fn = CI->getCalledFunction(); + if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { + Value *Exp = CI->getArgOperand(0); + CI->replaceAllUsesWith(Exp); + CI->eraseFromParent(); + } + } + } + + return false; +} + + +char LowerExpectIntrinsic::ID = 0; +INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", "Lower 'expect' " + "Intrinsics", false, false) + +FunctionPass *llvm::createLowerExpectIntrinsicPass() { + return new LowerExpectIntrinsic(); +} diff --git a/lib/Transforms/Utils/CMakeLists.txt b/lib/Transforms/Utils/CMakeLists.txt index 6ce22b10182..a30eee35fbd 100644 --- a/lib/Transforms/Utils/CMakeLists.txt +++ b/lib/Transforms/Utils/CMakeLists.txt @@ -21,7 +21,6 @@ add_llvm_library(LLVMTransformUtils LoopSimplify.cpp LoopUnroll.cpp LoopUnrollRuntime.cpp - LowerExpectIntrinsic.cpp LowerInvoke.cpp LowerSwitch.cpp Mem2Reg.cpp diff --git a/lib/Transforms/Utils/LowerExpectIntrinsic.cpp b/lib/Transforms/Utils/LowerExpectIntrinsic.cpp deleted file mode 100644 index ff89e747440..00000000000 --- a/lib/Transforms/Utils/LowerExpectIntrinsic.cpp +++ /dev/null @@ -1,188 +0,0 @@ -//===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This pass lowers the 'expect' intrinsic to LLVM metadata. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Transforms/Scalar.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/MDBuilder.h" -#include "llvm/IR/Metadata.h" -#include "llvm/Pass.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include - -using namespace llvm; - -#define DEBUG_TYPE "lower-expect-intrinsic" - -STATISTIC(IfHandled, "Number of 'expect' intrinsic instructions handled"); - -static cl::opt -LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64), - cl::desc("Weight of the branch likely to be taken (default = 64)")); -static cl::opt -UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4), - cl::desc("Weight of the branch unlikely to be taken (default = 4)")); - -namespace { - - class LowerExpectIntrinsic : public FunctionPass { - - bool HandleSwitchExpect(SwitchInst *SI); - - bool HandleIfExpect(BranchInst *BI); - - public: - static char ID; - LowerExpectIntrinsic() : FunctionPass(ID) { - initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); - } - - bool runOnFunction(Function &F) override; - }; -} - - -bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) { - CallInst *CI = dyn_cast(SI->getCondition()); - if (!CI) - return false; - - Function *Fn = CI->getCalledFunction(); - if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) - return false; - - Value *ArgValue = CI->getArgOperand(0); - ConstantInt *ExpectedValue = dyn_cast(CI->getArgOperand(1)); - if (!ExpectedValue) - return false; - - SwitchInst::CaseIt Case = SI->findCaseValue(ExpectedValue); - unsigned n = SI->getNumCases(); // +1 for default case. - std::vector Weights(n + 1); - - Weights[0] = Case == SI->case_default() ? LikelyBranchWeight - : UnlikelyBranchWeight; - for (unsigned i = 0; i != n; ++i) - Weights[i + 1] = i == Case.getCaseIndex() ? LikelyBranchWeight - : UnlikelyBranchWeight; - - SI->setMetadata(LLVMContext::MD_prof, - MDBuilder(CI->getContext()).createBranchWeights(Weights)); - - SI->setCondition(ArgValue); - return true; -} - - -bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) { - if (BI->isUnconditional()) - return false; - - // Handle non-optimized IR code like: - // %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1) - // %tobool = icmp ne i64 %expval, 0 - // br i1 %tobool, label %if.then, label %if.end - // - // Or the following simpler case: - // %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1) - // br i1 %expval, label %if.then, label %if.end - - CallInst *CI; - - ICmpInst *CmpI = dyn_cast(BI->getCondition()); - if (!CmpI) { - CI = dyn_cast(BI->getCondition()); - } else { - if (CmpI->getPredicate() != CmpInst::ICMP_NE) - return false; - CI = dyn_cast(CmpI->getOperand(0)); - } - - if (!CI) - return false; - - Function *Fn = CI->getCalledFunction(); - if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) - return false; - - Value *ArgValue = CI->getArgOperand(0); - ConstantInt *ExpectedValue = dyn_cast(CI->getArgOperand(1)); - if (!ExpectedValue) - return false; - - MDBuilder MDB(CI->getContext()); - MDNode *Node; - - // If expect value is equal to 1 it means that we are more likely to take - // branch 0, in other case more likely is branch 1. - if (ExpectedValue->isOne()) - Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); - else - Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); - - BI->setMetadata(LLVMContext::MD_prof, Node); - - if (CmpI) - CmpI->setOperand(0, ArgValue); - else - BI->setCondition(ArgValue); - return true; -} - - -bool LowerExpectIntrinsic::runOnFunction(Function &F) { - for (Function::iterator I = F.begin(), E = F.end(); I != E;) { - BasicBlock *BB = I++; - - // Create "block_weights" metadata. - if (BranchInst *BI = dyn_cast(BB->getTerminator())) { - if (HandleIfExpect(BI)) - IfHandled++; - } else if (SwitchInst *SI = dyn_cast(BB->getTerminator())) { - if (HandleSwitchExpect(SI)) - IfHandled++; - } - - // remove llvm.expect intrinsics. - for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); - BI != BE; ) { - CallInst *CI = dyn_cast(BI++); - if (!CI) - continue; - - Function *Fn = CI->getCalledFunction(); - if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { - Value *Exp = CI->getArgOperand(0); - CI->replaceAllUsesWith(Exp); - CI->eraseFromParent(); - } - } - } - - return false; -} - - -char LowerExpectIntrinsic::ID = 0; -INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", "Lower 'expect' " - "Intrinsics", false, false) - -FunctionPass *llvm::createLowerExpectIntrinsicPass() { - return new LowerExpectIntrinsic(); -}