typedef IRBuilder<true, TargetFolder> BuilderType;
BuilderType Builder;
+#ifndef NDEBUG
+ const char *DebugType;
+#endif
+
friend struct SCEVVisitor<SCEVExpander, Value*>;
public:
explicit SCEVExpander(ScalarEvolution &se, const char *name)
: SE(se), IVName(name), IVIncInsertLoop(0), IVIncInsertPos(0),
CanonicalMode(true), LSRMode(false),
- Builder(se.getContext(), TargetFolder(se.TD)) {}
+ Builder(se.getContext(), TargetFolder(se.TD)) {
+#ifndef NDEBUG
+ DebugType = "";
+#endif
+ }
+
+#ifndef NDEBUG
+ void setDebugType(const char* s) { DebugType = s; }
+#endif
/// clear - Erase the contents of the InsertedExpressions map so that users
/// trying to expand the same expression into multiple BasicBlocks or
/// starts at zero and steps by one on each iteration.
PHINode *getOrInsertCanonicalInductionVariable(const Loop *L, Type *Ty);
+ /// hoistStep - Utility for hoisting an IV increment.
+ static bool hoistStep(Instruction *IncV, Instruction *InsertPos,
+ const DominatorTree *DT);
+
+ /// replaceCongruentIVs - replace congruent phis with their most canonical
+ /// representative. Return the number of phis eliminated.
+ unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT,
+ SmallVectorImpl<WeakVH> &DeadInsts);
+
/// expandCodeFor - Insert code to directly compute the specified SCEV
/// expression into the program. The inserted code is inserted into the
/// specified block.
return V;
}
+
+/// hoistStep - Attempt to hoist an IV increment above a potential use.
+///
+/// To successfully hoist, two criteria must be met:
+/// - IncV operands dominate InsertPos and
+/// - InsertPos dominates IncV
+///
+/// Meeting the second condition means that we don't need to check all of IncV's
+/// existing uses (it's moving up in the domtree).
+///
+/// This does not yet recursively hoist the operands, although that would
+/// not be difficult.
+///
+/// This does not require a SCEVExpander instance and could be replaced by a
+/// general code-insertion helper.
+bool SCEVExpander::hoistStep(Instruction *IncV, Instruction *InsertPos,
+ const DominatorTree *DT) {
+ if (DT->dominates(IncV, InsertPos))
+ return true;
+
+ if (!DT->dominates(InsertPos->getParent(), IncV->getParent()))
+ return false;
+
+ if (IncV->mayHaveSideEffects())
+ return false;
+
+ // Attempt to hoist IncV
+ for (User::op_iterator OI = IncV->op_begin(), OE = IncV->op_end();
+ OI != OE; ++OI) {
+ Instruction *OInst = dyn_cast<Instruction>(OI);
+ if (OInst && !DT->dominates(OInst, InsertPos))
+ return false;
+ }
+ IncV->moveBefore(InsertPos);
+ return true;
+}
+
+/// replaceCongruentIVs - Check for congruent phis in this loop header and
+/// replace them with their most canonical representative. Return the number of
+/// phis eliminated.
+///
+/// This does not depend on any SCEVExpander state but should be used in
+/// the same context that SCEVExpander is used.
+unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
+ SmallVectorImpl<WeakVH> &DeadInsts) {
+ unsigned NumElim = 0;
+ DenseMap<const SCEV *, PHINode *> ExprToIVMap;
+ for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
+ PHINode *Phi = cast<PHINode>(I);
+ if (!SE.isSCEVable(Phi->getType()))
+ continue;
+
+ PHINode *&OrigPhiRef = ExprToIVMap[SE.getSCEV(Phi)];
+ if (!OrigPhiRef) {
+ OrigPhiRef = Phi;
+ continue;
+ }
+
+ // If one phi derives from the other via GEPs, types may differ.
+ // We could consider adding a bitcast here to handle it.
+ if (OrigPhiRef->getType() != Phi->getType())
+ continue;
+
+ if (BasicBlock *LatchBlock = L->getLoopLatch()) {
+ Instruction *OrigInc =
+ cast<Instruction>(OrigPhiRef->getIncomingValueForBlock(LatchBlock));
+ Instruction *IsomorphicInc =
+ cast<Instruction>(Phi->getIncomingValueForBlock(LatchBlock));
+
+ // If this phi is more canonical, swap it with the original.
+ if (!isExpandedAddRecExprPHI(OrigPhiRef, OrigInc, L,
+ OrigPhiRef->getType())
+ && isExpandedAddRecExprPHI(Phi, IsomorphicInc, L, Phi->getType())) {
+ std::swap(OrigPhiRef, Phi);
+ std::swap(OrigInc, IsomorphicInc);
+ }
+ // Replacing the congruent phi is sufficient because acyclic redundancy
+ // elimination, CSE/GVN, should handle the rest. However, once SCEV proves
+ // that a phi is congruent, it's often the head of an IV user cycle that
+ // is isomorphic with the original phi. So it's worth eagerly cleaning up
+ // the common case of a single IV increment.
+ if (OrigInc != IsomorphicInc &&
+ OrigInc->getType() == IsomorphicInc->getType() &&
+ SE.getSCEV(OrigInc) == SE.getSCEV(IsomorphicInc) &&
+ hoistStep(OrigInc, IsomorphicInc, DT)) {
+ DEBUG_WITH_TYPE(DebugType, dbgs()
+ << "INDVARS: Eliminated congruent iv.inc: "
+ << *IsomorphicInc << '\n');
+ IsomorphicInc->replaceAllUsesWith(OrigInc);
+ DeadInsts.push_back(IsomorphicInc);
+ }
+ }
+ DEBUG_WITH_TYPE(DebugType, dbgs()
+ << "INDVARS: Eliminated congruent iv: " << *Phi << '\n');
+ ++NumElim;
+ Phi->replaceAllUsesWith(OrigPhiRef);
+ DeadInsts.push_back(Phi);
+ }
+ return NumElim;
+}
void SimplifyAndExtend(Loop *L, SCEVExpander &Rewriter, LPPassManager &LPM);
- void SimplifyCongruentIVs(Loop *L);
-
void RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter);
void RewriteIVExpressions(Loop *L, SCEVExpander &Rewriter);
llvm_unreachable(0);
}
-/// HoistStep - Attempt to hoist an IV increment above a potential use.
-///
-/// To successfully hoist, two criteria must be met:
-/// - IncV operands dominate InsertPos and
-/// - InsertPos dominates IncV
-///
-/// Meeting the second condition means that we don't need to check all of IncV's
-/// existing uses (it's moving up in the domtree).
-///
-/// This does not yet recursively hoist the operands, although that would
-/// not be difficult.
-static bool HoistStep(Instruction *IncV, Instruction *InsertPos,
- const DominatorTree *DT)
-{
- if (DT->dominates(IncV, InsertPos))
- return true;
-
- if (!DT->dominates(InsertPos->getParent(), IncV->getParent()))
- return false;
-
- if (IncV->mayHaveSideEffects())
- return false;
-
- // Attempt to hoist IncV
- for (User::op_iterator OI = IncV->op_begin(), OE = IncV->op_end();
- OI != OE; ++OI) {
- Instruction *OInst = dyn_cast<Instruction>(OI);
- if (OInst && !DT->dominates(OInst, InsertPos))
- return false;
- }
- IncV->moveBefore(InsertPos);
- return true;
-}
-
/// No-wrap operations can transfer sign extension of their result to their
/// operands. Generate the SCEV value for the widened operation without
/// actually modifying the IR yet. If the expression after extending the
// Reuse the IV increment that SCEVExpander created as long as it dominates
// NarrowUse.
Instruction *WideUse = 0;
- if (WideAddRec == WideIncExpr && HoistStep(WideInc, DU.NarrowUse, DT)) {
+ if (WideAddRec == WideIncExpr
+ && SCEVExpander::hoistStep(WideInc, DU.NarrowUse, DT))
WideUse = WideInc;
- }
else {
WideUse = CloneIVUser(DU);
if (!WideUse)
}
}
-/// SimplifyCongruentIVs - Check for congruent phis in this loop header and
-/// replace them with their chosen representative.
-///
-void IndVarSimplify::SimplifyCongruentIVs(Loop *L) {
- DenseMap<const SCEV *, PHINode *> ExprToIVMap;
- for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
- PHINode *Phi = cast<PHINode>(I);
- if (!SE->isSCEVable(Phi->getType()))
- continue;
-
- const SCEV *S = SE->getSCEV(Phi);
- std::pair<DenseMap<const SCEV *, PHINode *>::const_iterator, bool> Tmp =
- ExprToIVMap.insert(std::make_pair(S, Phi));
- if (Tmp.second)
- continue;
- PHINode *OrigPhi = Tmp.first->second;
-
- // If one phi derives from the other via GEPs, types may differ.
- if (OrigPhi->getType() != Phi->getType())
- continue;
-
- // Replacing the congruent phi is sufficient because acyclic redundancy
- // elimination, CSE/GVN, should handle the rest. However, once SCEV proves
- // that a phi is congruent, it's almost certain to be the head of an IV
- // user cycle that is isomorphic with the original phi. So it's worth
- // eagerly cleaning up the common case of a single IV increment.
- if (BasicBlock *LatchBlock = L->getLoopLatch()) {
- Instruction *OrigInc =
- cast<Instruction>(OrigPhi->getIncomingValueForBlock(LatchBlock));
- Instruction *IsomorphicInc =
- cast<Instruction>(Phi->getIncomingValueForBlock(LatchBlock));
- if (OrigInc != IsomorphicInc &&
- OrigInc->getType() == IsomorphicInc->getType() &&
- SE->getSCEV(OrigInc) == SE->getSCEV(IsomorphicInc) &&
- HoistStep(OrigInc, IsomorphicInc, DT)) {
- DEBUG(dbgs() << "INDVARS: Eliminated congruent iv.inc: "
- << *IsomorphicInc << '\n');
- IsomorphicInc->replaceAllUsesWith(OrigInc);
- DeadInsts.push_back(IsomorphicInc);
- }
- }
- DEBUG(dbgs() << "INDVARS: Eliminated congruent iv: " << *Phi << '\n');
- ++NumElimIV;
- Phi->replaceAllUsesWith(OrigPhi);
- DeadInsts.push_back(Phi);
- }
-}
-
//===----------------------------------------------------------------------===//
// LinearFunctionTestReplace and its kin. Rewrite the loop exit condition.
//===----------------------------------------------------------------------===//
// Create a rewriter object which we'll use to transform the code with.
SCEVExpander Rewriter(*SE, "indvars");
+#ifndef NDEBUG
+ Rewriter.setDebugType(DEBUG_TYPE);
+#endif
// Eliminate redundant IV users.
//
// Eliminate redundant IV cycles.
if (!EnableIVRewrite)
- SimplifyCongruentIVs(L);
+ NumElimIV += Rewriter.replaceCongruentIVs(L, DT, DeadInsts);
// Compute the type of the largest recurrence expression, and decide whether
// a canonical induction variable should be inserted.
; CHECK-NOT: phi
; CHECK: add i32
; CHECK: add i32
+; CHECK: add i32
; CHECK-NOT: add
; CHECK: return:
;
projects / oota-llvm.git / commitdiff