X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FTransforms%2FVectorize%2FBBVectorize.cpp;h=8844d574a79d320f2c2a8e431b6aa473f0be1f69;hb=8bb7acb4c17874dd85a9acd009295c5d5cb0fbf9;hp=29fb01f1b2ec79444cace5c61d50f1cb22699634;hpb=529919ff310cbfce1ba55ea252ff738d5b56b93d;p=oota-llvm.git diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 29fb01f1b2e..8844d574a79 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -25,8 +25,11 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AliasSetTracker.h" +#include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Constants.h" @@ -204,9 +207,10 @@ namespace { BBVectorize(Pass *P, Function &F, const VectorizeConfig &C) : BasicBlockPass(ID), Config(C) { - AA = &P->getAnalysis(); + AA = &P->getAnalysis().getAAResults(); DT = &P->getAnalysis().getDomTree(); - SE = &P->getAnalysis(); + SE = &P->getAnalysis().getSE(); + TLI = &P->getAnalysis().getTLI(); TTI = IgnoreTargetInfo ? nullptr : &P->getAnalysis().getTTI(F); @@ -221,6 +225,7 @@ namespace { AliasAnalysis *AA; DominatorTree *DT; ScalarEvolution *SE; + const TargetLibraryInfo *TLI; const TargetTransformInfo *TTI; // FIXME: const correct? @@ -437,9 +442,10 @@ namespace { bool runOnBasicBlock(BasicBlock &BB) override { // OptimizeNone check deferred to vectorizeBB(). - AA = &getAnalysis(); + AA = &getAnalysis().getAAResults(); DT = &getAnalysis().getDomTree(); - SE = &getAnalysis(); + SE = &getAnalysis().getSE(); + TLI = &getAnalysis().getTLI(); TTI = IgnoreTargetInfo ? nullptr : &getAnalysis().getTTI( @@ -450,13 +456,15 @@ namespace { void getAnalysisUsage(AnalysisUsage &AU) const override { BasicBlockPass::getAnalysisUsage(AU); - AU.addRequired(); + AU.addRequired(); AU.addRequired(); - AU.addRequired(); + AU.addRequired(); + AU.addRequired(); AU.addRequired(); - AU.addPreserved(); AU.addPreserved(); - AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); AU.setPreservesCFG(); } @@ -662,7 +670,7 @@ namespace { Function *F = I->getCalledFunction(); if (!F) return false; - Intrinsic::ID IID = (Intrinsic::ID) F->getIntrinsicID(); + Intrinsic::ID IID = F->getIntrinsicID(); if (!IID) return false; switch(IID) { @@ -842,7 +850,7 @@ namespace { // It is important to cleanup here so that future iterations of this // function have less work to do. - (void)SimplifyInstructionsInBlock(&BB, AA->getTargetLibraryInfo()); + (void)SimplifyInstructionsInBlock(&BB, TLI); return true; } @@ -1098,7 +1106,7 @@ namespace { CallInst *CI = dyn_cast(I); Function *FI; if (CI && (FI = CI->getCalledFunction())) { - Intrinsic::ID IID = (Intrinsic::ID) FI->getIntrinsicID(); + Intrinsic::ID IID = FI->getIntrinsicID(); if (IID == Intrinsic::powi || IID == Intrinsic::ctlz || IID == Intrinsic::cttz) { Value *A1I = CI->getArgOperand(1), @@ -1239,20 +1247,23 @@ namespace { if (I == Start) IAfterStart = true; bool IsSimpleLoadStore; - if (!isInstVectorizable(I, IsSimpleLoadStore)) continue; + if (!isInstVectorizable(&*I, IsSimpleLoadStore)) + continue; // Look for an instruction with which to pair instruction *I... DenseSet Users; AliasSetTracker WriteSet(*AA); - if (I->mayWriteToMemory()) WriteSet.add(I); + if (I->mayWriteToMemory()) + WriteSet.add(&*I); bool JAfterStart = IAfterStart; BasicBlock::iterator J = std::next(I); for (unsigned ss = 0; J != E && ss <= Config.SearchLimit; ++J, ++ss) { - if (J == Start) JAfterStart = true; + if (&*J == Start) + JAfterStart = true; // Determine if J uses I, if so, exit the loop. - bool UsesI = trackUsesOfI(Users, WriteSet, I, J, !Config.FastDep); + bool UsesI = trackUsesOfI(Users, WriteSet, &*I, &*J, !Config.FastDep); if (Config.FastDep) { // Note: For this heuristic to be effective, independent operations // must tend to be intermixed. This is likely to be true from some @@ -1269,25 +1280,26 @@ namespace { // J does not use I, and comes before the first use of I, so it can be // merged with I if the instructions are compatible. int CostSavings, FixedOrder; - if (!areInstsCompatible(I, J, IsSimpleLoadStore, NonPow2Len, - CostSavings, FixedOrder)) continue; + if (!areInstsCompatible(&*I, &*J, IsSimpleLoadStore, NonPow2Len, + CostSavings, FixedOrder)) + continue; // J is a candidate for merging with I. if (PairableInsts.empty() || - PairableInsts[PairableInsts.size()-1] != I) { - PairableInsts.push_back(I); + PairableInsts[PairableInsts.size() - 1] != &*I) { + PairableInsts.push_back(&*I); } - CandidatePairs[I].push_back(J); + CandidatePairs[&*I].push_back(&*J); ++TotalPairs; if (TTI) - CandidatePairCostSavings.insert(ValuePairWithCost(ValuePair(I, J), - CostSavings)); + CandidatePairCostSavings.insert( + ValuePairWithCost(ValuePair(&*I, &*J), CostSavings)); if (FixedOrder == 1) - FixedOrderPairs.insert(ValuePair(I, J)); + FixedOrderPairs.insert(ValuePair(&*I, &*J)); else if (FixedOrder == -1) - FixedOrderPairs.insert(ValuePair(J, I)); + FixedOrderPairs.insert(ValuePair(&*J, &*I)); // The next call to this function must start after the last instruction // selected during this invocation. @@ -1468,14 +1480,16 @@ namespace { BasicBlock::iterator E = BB.end(), EL = BasicBlock::iterator(cast(PairableInsts.back())); for (BasicBlock::iterator I = BB.getFirstInsertionPt(); I != E; ++I) { - if (IsInPair.find(I) == IsInPair.end()) continue; + if (IsInPair.find(&*I) == IsInPair.end()) + continue; DenseSet Users; AliasSetTracker WriteSet(*AA); - if (I->mayWriteToMemory()) WriteSet.add(I); + if (I->mayWriteToMemory()) + WriteSet.add(&*I); for (BasicBlock::iterator J = std::next(I); J != E; ++J) { - (void) trackUsesOfI(Users, WriteSet, I, J); + (void)trackUsesOfI(Users, WriteSet, &*I, &*J); if (J == EL) break; @@ -1484,7 +1498,7 @@ namespace { for (DenseSet::iterator U = Users.begin(), E = Users.end(); U != E; ++U) { if (IsInPair.find(*U) == IsInPair.end()) continue; - PairableInstUsers.insert(ValuePair(I, *U)); + PairableInstUsers.insert(ValuePair(&*I, *U)); } if (I == EL) @@ -2770,7 +2784,7 @@ namespace { continue; } else if (isa(I)) { Function *F = cast(I)->getCalledFunction(); - Intrinsic::ID IID = (Intrinsic::ID) F->getIntrinsicID(); + Intrinsic::ID IID = F->getIntrinsicID(); if (o == NumOperands-1) { BasicBlock &BB = *I->getParent(); @@ -2806,55 +2820,51 @@ namespace { Instruction *J, Instruction *K, Instruction *&InsertionPt, Instruction *&K1, Instruction *&K2) { - if (isa(I)) { - AA->replaceWithNewValue(I, K); - AA->replaceWithNewValue(J, K); - } else { - Type *IType = I->getType(); - Type *JType = J->getType(); + if (isa(I)) + return; - VectorType *VType = getVecTypeForPair(IType, JType); - unsigned numElem = VType->getNumElements(); + Type *IType = I->getType(); + Type *JType = J->getType(); - unsigned numElemI = getNumScalarElements(IType); - unsigned numElemJ = getNumScalarElements(JType); + VectorType *VType = getVecTypeForPair(IType, JType); + unsigned numElem = VType->getNumElements(); - if (IType->isVectorTy()) { - std::vector Mask1(numElemI), Mask2(numElemI); - for (unsigned v = 0; v < numElemI; ++v) { - Mask1[v] = ConstantInt::get(Type::getInt32Ty(Context), v); - Mask2[v] = ConstantInt::get(Type::getInt32Ty(Context), numElemJ+v); - } + unsigned numElemI = getNumScalarElements(IType); + unsigned numElemJ = getNumScalarElements(JType); - K1 = new ShuffleVectorInst(K, UndefValue::get(VType), - ConstantVector::get( Mask1), - getReplacementName(K, false, 1)); - } else { - Value *CV0 = ConstantInt::get(Type::getInt32Ty(Context), 0); - K1 = ExtractElementInst::Create(K, CV0, - getReplacementName(K, false, 1)); + if (IType->isVectorTy()) { + std::vector Mask1(numElemI), Mask2(numElemI); + for (unsigned v = 0; v < numElemI; ++v) { + Mask1[v] = ConstantInt::get(Type::getInt32Ty(Context), v); + Mask2[v] = ConstantInt::get(Type::getInt32Ty(Context), numElemJ + v); } - if (JType->isVectorTy()) { - std::vector Mask1(numElemJ), Mask2(numElemJ); - for (unsigned v = 0; v < numElemJ; ++v) { - Mask1[v] = ConstantInt::get(Type::getInt32Ty(Context), v); - Mask2[v] = ConstantInt::get(Type::getInt32Ty(Context), numElemI+v); - } + K1 = new ShuffleVectorInst(K, UndefValue::get(VType), + ConstantVector::get(Mask1), + getReplacementName(K, false, 1)); + } else { + Value *CV0 = ConstantInt::get(Type::getInt32Ty(Context), 0); + K1 = ExtractElementInst::Create(K, CV0, getReplacementName(K, false, 1)); + } - K2 = new ShuffleVectorInst(K, UndefValue::get(VType), - ConstantVector::get( Mask2), - getReplacementName(K, false, 2)); - } else { - Value *CV1 = ConstantInt::get(Type::getInt32Ty(Context), numElem-1); - K2 = ExtractElementInst::Create(K, CV1, - getReplacementName(K, false, 2)); + if (JType->isVectorTy()) { + std::vector Mask1(numElemJ), Mask2(numElemJ); + for (unsigned v = 0; v < numElemJ; ++v) { + Mask1[v] = ConstantInt::get(Type::getInt32Ty(Context), v); + Mask2[v] = ConstantInt::get(Type::getInt32Ty(Context), numElemI + v); } - K1->insertAfter(K); - K2->insertAfter(K1); - InsertionPt = K2; + K2 = new ShuffleVectorInst(K, UndefValue::get(VType), + ConstantVector::get(Mask2), + getReplacementName(K, false, 2)); + } else { + Value *CV1 = ConstantInt::get(Type::getInt32Ty(Context), numElem - 1); + K2 = ExtractElementInst::Create(K, CV1, getReplacementName(K, false, 2)); } + + K1->insertAfter(K); + K2->insertAfter(K1); + InsertionPt = K2; } // Move all uses of the function I (including pairing-induced uses) after J. @@ -2869,7 +2879,7 @@ namespace { if (I->mayWriteToMemory()) WriteSet.add(I); for (; cast(L) != J; ++L) - (void) trackUsesOfI(Users, WriteSet, I, L, true, &LoadMoveSetPairs); + (void)trackUsesOfI(Users, WriteSet, I, &*L, true, &LoadMoveSetPairs); assert(cast(L) == J && "Tracking has not proceeded far enough to check for dependencies"); @@ -2891,9 +2901,9 @@ namespace { if (I->mayWriteToMemory()) WriteSet.add(I); for (; cast(L) != J;) { - if (trackUsesOfI(Users, WriteSet, I, L, true, &LoadMoveSetPairs)) { + if (trackUsesOfI(Users, WriteSet, I, &*L, true, &LoadMoveSetPairs)) { // Move this instruction - Instruction *InstToMove = L; ++L; + Instruction *InstToMove = &*L++; DEBUG(dbgs() << "BBV: moving: " << *InstToMove << " to after " << *InsertionPt << "\n"); @@ -2924,11 +2934,11 @@ namespace { // Note: We cannot end the loop when we reach J because J could be moved // farther down the use chain by another instruction pairing. Also, J // could be before I if this is an inverted input. - for (BasicBlock::iterator E = BB.end(); cast(L) != E; ++L) { - if (trackUsesOfI(Users, WriteSet, I, L)) { + for (BasicBlock::iterator E = BB.end(); L != E; ++L) { + if (trackUsesOfI(Users, WriteSet, I, &*L)) { if (L->mayReadFromMemory()) { - LoadMoveSet[L].push_back(I); - LoadMoveSetPairs.insert(ValuePair(L, I)); + LoadMoveSet[&*L].push_back(I); + LoadMoveSetPairs.insert(ValuePair(&*L, I)); } } } @@ -2991,7 +3001,7 @@ namespace { DEBUG(dbgs() << "BBV: initial: \n" << BB << "\n"); for (BasicBlock::iterator PI = BB.getFirstInsertionPt(); PI != BB.end();) { - DenseMap::iterator P = ChosenPairs.find(PI); + DenseMap::iterator P = ChosenPairs.find(&*PI); if (P == ChosenPairs.end()) { ++PI; continue; @@ -3103,15 +3113,22 @@ namespace { else if (H->hasName()) K->takeName(H); - if (!isa(K)) + if (auto CS = CallSite(K)) { + SmallVector Tys; + FunctionType *Old = CS.getFunctionType(); + unsigned NumOld = Old->getNumParams(); + assert(NumOld <= ReplacedOperands.size()); + for (unsigned i = 0; i != NumOld; ++i) + Tys.push_back(ReplacedOperands[i]->getType()); + CS.mutateFunctionType( + FunctionType::get(getVecTypeForPair(L->getType(), H->getType()), + Tys, Old->isVarArg())); + } else if (!isa(K)) K->mutateType(getVecTypeForPair(L->getType(), H->getType())); - unsigned KnownIDs[] = { - LLVMContext::MD_tbaa, - LLVMContext::MD_alias_scope, - LLVMContext::MD_noalias, - LLVMContext::MD_fpmath - }; + unsigned KnownIDs[] = {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope, + LLVMContext::MD_noalias, LLVMContext::MD_fpmath, + LLVMContext::MD_invariant_group}; combineMetadata(K, H, KnownIDs); K->intersectOptionalDataWith(H); @@ -3135,8 +3152,6 @@ namespace { if (!isa(I)) { L->replaceAllUsesWith(K1); H->replaceAllUsesWith(K2); - AA->replaceWithNewValue(L, K1); - AA->replaceWithNewValue(H, K2); } // Instructions that may read from memory may be in the load move set. @@ -3187,10 +3202,14 @@ namespace { char BBVectorize::ID = 0; static const char bb_vectorize_name[] = "Basic-Block Vectorization"; INITIALIZE_PASS_BEGIN(BBVectorize, BBV_NAME, bb_vectorize_name, false, false) -INITIALIZE_AG_DEPENDENCY(AliasAnalysis) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) -INITIALIZE_PASS_DEPENDENCY(ScalarEvolution) +INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) +INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass) +INITIALIZE_PASS_DEPENDENCY(SCEVAAWrapperPass) INITIALIZE_PASS_END(BBVectorize, BBV_NAME, bb_vectorize_name, false, false) BasicBlockPass *llvm::createBBVectorizePass(const VectorizeConfig &C) {