X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FTailDuplication.cpp;h=ca929d9fa4504f5b80f8ec872c40147839de5994;hb=919f1f47e4211b21877d74b0543ec53bf4a165c1;hp=3cf793478093c4d7e565bd091a4fc7d52ba11c51;hpb=831737d329a727f53a1fb0572f7b7a8127208881;p=oota-llvm.git diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp index 3cf79347809..ca929d9fa45 100644 --- a/lib/CodeGen/TailDuplication.cpp +++ b/lib/CodeGen/TailDuplication.cpp @@ -12,28 +12,30 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "tailduplication" #include "llvm/CodeGen/Passes.h" #include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/OwningPtr.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineSSAUpdater.h" #include "llvm/CodeGen/RegisterScavenging.h" -#include "llvm/Function.h" +#include "llvm/IR/Function.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; +#define DEBUG_TYPE "tailduplication" + STATISTIC(NumTails , "Number of tails duplicated"); STATISTIC(NumTailDups , "Number of tail duplicated blocks"); STATISTIC(NumInstrDups , "Additional instructions due to tail duplication"); @@ -57,20 +59,21 @@ TailDupLimit("tail-dup-limit", cl::init(~0U), cl::Hidden); typedef std::vector > AvailableValsTy; namespace { - /// TailDuplicatePass - Perform tail duplication. + /// Perform tail duplication. class TailDuplicatePass : public MachineFunctionPass { const TargetInstrInfo *TII; const TargetRegisterInfo *TRI; + const MachineBranchProbabilityInfo *MBPI; MachineModuleInfo *MMI; MachineRegisterInfo *MRI; - OwningPtr RS; + std::unique_ptr RS; bool PreRegAlloc; - // SSAUpdateVRs - A list of virtual registers for which to update SSA form. + // A list of virtual registers for which to update SSA form. SmallVector SSAUpdateVRs; - // SSAUpdateVals - For each virtual register in SSAUpdateVals keep a list of - // source virtual registers. + // For each virtual register in SSAUpdateVals keep a list of source virtual + // registers. DenseMap SSAUpdateVals; public: @@ -78,7 +81,9 @@ namespace { explicit TailDuplicatePass() : MachineFunctionPass(ID), PreRegAlloc(false) {} - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override; private: void AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg, @@ -86,7 +91,7 @@ namespace { void ProcessPHI(MachineInstr *MI, MachineBasicBlock *TailBB, MachineBasicBlock *PredBB, DenseMap &LocalVRMap, - SmallVector, 4> &Copies, + SmallVectorImpl > &Copies, const DenseSet &UsedByPhi, bool Remove); void DuplicateInstruction(MachineInstr *MI, @@ -96,7 +101,7 @@ namespace { DenseMap &LocalVRMap, const DenseSet &UsedByPhi); void UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, - SmallVector &TDBBs, + SmallVectorImpl &TDBBs, SmallSetVector &Succs); bool TailDuplicateBlocks(MachineFunction &MF); bool shouldTailDuplicate(const MachineFunction &MF, @@ -104,14 +109,14 @@ namespace { bool isSimpleBB(MachineBasicBlock *TailBB); bool canCompletelyDuplicateBB(MachineBasicBlock &BB); bool duplicateSimpleBB(MachineBasicBlock *TailBB, - SmallVector &TDBBs, + SmallVectorImpl &TDBBs, const DenseSet &RegsUsedByPhi, - SmallVector &Copies); + SmallVectorImpl &Copies); bool TailDuplicate(MachineBasicBlock *TailBB, bool IsSimple, MachineFunction &MF, - SmallVector &TDBBs, - SmallVector &Copies); + SmallVectorImpl &TDBBs, + SmallVectorImpl &Copies); bool TailDuplicateAndUpdate(MachineBasicBlock *MBB, bool IsSimple, MachineFunction &MF); @@ -128,10 +133,15 @@ INITIALIZE_PASS(TailDuplicatePass, "tailduplication", "Tail Duplication", false, false) bool TailDuplicatePass::runOnMachineFunction(MachineFunction &MF) { - TII = MF.getTarget().getInstrInfo(); - TRI = MF.getTarget().getRegisterInfo(); + if (skipOptnoneFunction(*MF.getFunction())) + return false; + + TII = MF.getSubtarget().getInstrInfo(); + TRI = MF.getSubtarget().getRegisterInfo(); MRI = &MF.getRegInfo(); MMI = getAnalysisIfAvailable(); + MBPI = &getAnalysis(); + PreRegAlloc = MRI->isSSA(); RS.reset(); if (MRI->tracksLiveness() && TRI->trackLivenessAfterRegAlloc(MF)) @@ -144,6 +154,11 @@ bool TailDuplicatePass::runOnMachineFunction(MachineFunction &MF) { return MadeChange; } +void TailDuplicatePass::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); + MachineFunctionPass::getAnalysisUsage(AU); +} + static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ++I) { MachineBasicBlock *MBB = I; @@ -168,7 +183,7 @@ static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; dbgs() << " missing input from predecessor BB#" << PredBB->getNumber() << '\n'; - llvm_unreachable(0); + llvm_unreachable(nullptr); } } @@ -179,12 +194,12 @@ static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { << ": " << *MI; dbgs() << " extra input from predecessor BB#" << PHIBB->getNumber() << '\n'; - llvm_unreachable(0); + llvm_unreachable(nullptr); } if (PHIBB->getNumber() < 0) { dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; dbgs() << " non-existing BB#" << PHIBB->getNumber() << '\n'; - llvm_unreachable(0); + llvm_unreachable(nullptr); } } ++MI; @@ -192,7 +207,7 @@ static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { } } -/// TailDuplicateAndUpdate - Tail duplicate the block and cleanup. +/// Tail duplicate the block and cleanup. bool TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB, bool IsSimple, @@ -234,7 +249,7 @@ TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB, // If the original definition is still around, add it as an available // value. MachineInstr *DefMI = MRI->getVRegDef(VReg); - MachineBasicBlock *DefBB = 0; + MachineBasicBlock *DefBB = nullptr; if (DefMI) { DefBB = DefMI->getParent(); SSAUpdate.AddAvailableValue(DefBB, VReg); @@ -252,8 +267,8 @@ TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB, // Rewrite uses that are outside of the original def's block. MachineRegisterInfo::use_iterator UI = MRI->use_begin(VReg); while (UI != MRI->use_end()) { - MachineOperand &UseMO = UI.getOperand(); - MachineInstr *UseMI = &*UI; + MachineOperand &UseMO = *UI; + MachineInstr *UseMI = UseMO.getParent(); ++UI; if (UseMI->isDebugValue()) { // SSAUpdate can replace the use with an undef. That creates @@ -295,9 +310,9 @@ TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB, return true; } -/// TailDuplicateBlocks - Look for small blocks that are unconditionally -/// branched to and do not fall through. Tail-duplicate their instructions -/// into their predecessors to eliminate (dynamic) branches. +/// Look for small blocks that are unconditionally branched to and do not fall +/// through. Tail-duplicate their instructions into their predecessors to +/// eliminate (dynamic) branches. bool TailDuplicatePass::TailDuplicateBlocks(MachineFunction &MF) { bool MadeChange = false; @@ -328,12 +343,10 @@ bool TailDuplicatePass::TailDuplicateBlocks(MachineFunction &MF) { static bool isDefLiveOut(unsigned Reg, MachineBasicBlock *BB, const MachineRegisterInfo *MRI) { - for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(Reg), - UE = MRI->use_end(); UI != UE; ++UI) { - MachineInstr *UseMI = &*UI; - if (UseMI->isDebugValue()) + for (MachineInstr &UseMI : MRI->use_instructions(Reg)) { + if (UseMI.isDebugValue()) continue; - if (UseMI->getParent() != BB) + if (UseMI.getParent() != BB) return true; } return false; @@ -352,9 +365,7 @@ static unsigned getPHISrcRegOpIdx(MachineInstr *MI, MachineBasicBlock *SrcBB) { // block (which is why we need to copy the information). static void getRegsUsedByPHIs(const MachineBasicBlock &BB, DenseSet *UsedByPhi) { - for(MachineBasicBlock::const_iterator I = BB.begin(), E = BB.end(); - I != E; ++I) { - const MachineInstr &MI = *I; + for (const auto &MI : BB) { if (!MI.isPHI()) break; for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { @@ -364,8 +375,7 @@ static void getRegsUsedByPHIs(const MachineBasicBlock &BB, } } -/// AddSSAUpdateEntry - Add a definition and source virtual registers pair for -/// SSA update. +/// Add a definition and source virtual registers pair for SSA update. void TailDuplicatePass::AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg, MachineBasicBlock *BB) { DenseMap::iterator LI= SSAUpdateVals.find(OrigReg); @@ -379,16 +389,13 @@ void TailDuplicatePass::AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg, } } -/// ProcessPHI - Process PHI node in TailBB by turning it into a copy in PredBB. -/// Remember the source register that's contributed by PredBB and update SSA -/// update map. -void TailDuplicatePass::ProcessPHI(MachineInstr *MI, - MachineBasicBlock *TailBB, - MachineBasicBlock *PredBB, - DenseMap &LocalVRMap, - SmallVector, 4> &Copies, - const DenseSet &RegsUsedByPhi, - bool Remove) { +/// Process PHI node in TailBB by turning it into a copy in PredBB. Remember the +/// source register that's contributed by PredBB and update SSA update map. +void TailDuplicatePass::ProcessPHI( + MachineInstr *MI, MachineBasicBlock *TailBB, MachineBasicBlock *PredBB, + DenseMap &LocalVRMap, + SmallVectorImpl > &Copies, + const DenseSet &RegsUsedByPhi, bool Remove) { unsigned DefReg = MI->getOperand(0).getReg(); unsigned SrcOpIdx = getPHISrcRegOpIdx(MI, PredBB); assert(SrcOpIdx && "Unable to find matching PHI source?"); @@ -413,7 +420,7 @@ void TailDuplicatePass::ProcessPHI(MachineInstr *MI, MI->eraseFromParent(); } -/// DuplicateInstruction - Duplicate a TailBB instruction to PredBB and update +/// Duplicate a TailBB instruction to PredBB and update /// the source operands due to earlier PHI translation. void TailDuplicatePass::DuplicateInstruction(MachineInstr *MI, MachineBasicBlock *TailBB, @@ -440,6 +447,9 @@ void TailDuplicatePass::DuplicateInstruction(MachineInstr *MI, DenseMap::iterator VI = LocalVRMap.find(Reg); if (VI != LocalVRMap.end()) { MO.setReg(VI->second); + // Clear any kill flags from this operand. The new register could have + // uses after this one, so kills are not valid here. + MO.setIsKill(false); MRI->constrainRegClass(VI->second, MRI->getRegClass(Reg)); } } @@ -447,12 +457,12 @@ void TailDuplicatePass::DuplicateInstruction(MachineInstr *MI, PredBB->insert(PredBB->instr_end(), NewMI); } -/// UpdateSuccessorsPHIs - After FromBB is tail duplicated into its predecessor -/// blocks, the successors have gained new predecessors. Update the PHI -/// instructions in them accordingly. +/// After FromBB is tail duplicated into its predecessor blocks, the successors +/// have gained new predecessors. Update the PHI instructions in them +/// accordingly. void TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, - SmallVector &TDBBs, + SmallVectorImpl &TDBBs, SmallSetVector &Succs) { for (SmallSetVector::iterator SI = Succs.begin(), SE = Succs.end(); SI != SE; ++SI) { @@ -533,7 +543,7 @@ TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, } } -/// shouldTailDuplicate - Determine if it is profitable to duplicate this block. +/// Determine if it is profitable to duplicate this block. bool TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, bool IsSimple, @@ -551,8 +561,8 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, // compensate for the duplication. unsigned MaxDuplicateCount; if (TailDuplicateSize.getNumOccurrences() == 0 && - MF.getFunction()->getAttributes(). - hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize)) + // FIXME: Use Function::optForSize(). + MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize)) MaxDuplicateCount = 1; else MaxDuplicateCount = TailDuplicateSize; @@ -573,24 +583,24 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, // Check the instructions in the block to determine whether tail-duplication // is invalid or unlikely to be profitable. unsigned InstrCount = 0; - for (MachineBasicBlock::iterator I = TailBB.begin(); I != TailBB.end(); ++I) { + for (MachineInstr &MI : TailBB) { // Non-duplicable things shouldn't be tail-duplicated. - if (I->isNotDuplicable()) + if (MI.isNotDuplicable()) return false; // Do not duplicate 'return' instructions if this is a pre-regalloc run. // A return may expand into a lot more instructions (e.g. reload of callee // saved registers) after PEI. - if (PreRegAlloc && I->isReturn()) + if (PreRegAlloc && MI.isReturn()) return false; // Avoid duplicating calls before register allocation. Calls presents a // barrier to register allocation so duplicating them may end up increasing // spills. - if (PreRegAlloc && I->isCall()) + if (PreRegAlloc && MI.isCall()) return false; - if (!I->isPHI() && !I->isDebugValue()) + if (!MI.isPHI() && !MI.isDebugValue()) InstrCount += 1; if (InstrCount > MaxDuplicateCount) @@ -609,18 +619,15 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, return canCompletelyDuplicateBB(TailBB); } -/// isSimpleBB - True if this BB has only one unconditional jump. +/// True if this BB has only one unconditional jump. bool TailDuplicatePass::isSimpleBB(MachineBasicBlock *TailBB) { if (TailBB->succ_size() != 1) return false; if (TailBB->pred_empty()) return false; - MachineBasicBlock::iterator I = TailBB->begin(); - MachineBasicBlock::iterator E = TailBB->end(); - while (I != E && I->isDebugValue()) - ++I; - if (I == E) + MachineBasicBlock::iterator I = TailBB->getFirstNonDebugInstr(); + if (I == TailBB->end()) return true; return I->isUnconditionalBranch(); } @@ -628,28 +635,20 @@ TailDuplicatePass::isSimpleBB(MachineBasicBlock *TailBB) { static bool bothUsedInPHI(const MachineBasicBlock &A, SmallPtrSet SuccsB) { - for (MachineBasicBlock::const_succ_iterator SI = A.succ_begin(), - SE = A.succ_end(); SI != SE; ++SI) { - MachineBasicBlock *BB = *SI; + for (MachineBasicBlock *BB : A.successors()) if (SuccsB.count(BB) && !BB->empty() && BB->begin()->isPHI()) return true; - } return false; } bool TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) { - SmallPtrSet Succs(BB.succ_begin(), BB.succ_end()); - - for (MachineBasicBlock::pred_iterator PI = BB.pred_begin(), - PE = BB.pred_end(); PI != PE; ++PI) { - MachineBasicBlock *PredBB = *PI; - + for (MachineBasicBlock *PredBB : BB.predecessors()) { if (PredBB->succ_size() > 1) return false; - MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; + MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr; SmallVector PredCond; if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) return false; @@ -662,9 +661,9 @@ TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) { bool TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, - SmallVector &TDBBs, - const DenseSet &UsedByPhi, - SmallVector &Copies) { + SmallVectorImpl &TDBBs, + const DenseSet &UsedByPhi, + SmallVectorImpl &Copies) { SmallPtrSet Succs(TailBB->succ_begin(), TailBB->succ_end()); SmallVector Preds(TailBB->pred_begin(), @@ -680,7 +679,7 @@ TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, if (bothUsedInPHI(*PredBB, Succs)) continue; - MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; + MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr; SmallVector PredCond; if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) continue; @@ -690,7 +689,7 @@ TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, << "From simple Succ: " << *TailBB); MachineBasicBlock *NewTarget = *TailBB->succ_begin(); - MachineBasicBlock *NextBB = llvm::next(MachineFunction::iterator(PredBB)); + MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(PredBB)); // Make PredFBB explicit. if (PredCond.empty()) @@ -711,39 +710,40 @@ TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, // Make the branch unconditional if possible if (PredTBB == PredFBB) { PredCond.clear(); - PredFBB = NULL; + PredFBB = nullptr; } // Avoid adding fall through branches. if (PredFBB == NextBB) - PredFBB = NULL; - if (PredTBB == NextBB && PredFBB == NULL) - PredTBB = NULL; + PredFBB = nullptr; + if (PredTBB == NextBB && PredFBB == nullptr) + PredTBB = nullptr; TII->RemoveBranch(*PredBB); if (PredTBB) TII->InsertBranch(*PredBB, PredTBB, PredFBB, PredCond, DebugLoc()); + uint32_t Weight = MBPI->getEdgeWeight(PredBB, TailBB); PredBB->removeSuccessor(TailBB); unsigned NumSuccessors = PredBB->succ_size(); assert(NumSuccessors <= 1); if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget) - PredBB->addSuccessor(NewTarget); + PredBB->addSuccessor(NewTarget, Weight); TDBBs.push_back(PredBB); } return Changed; } -/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each +/// If it is profitable, duplicate TailBB's contents in each /// of its predecessors. bool TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, bool IsSimple, MachineFunction &MF, - SmallVector &TDBBs, - SmallVector &Copies) { + SmallVectorImpl &TDBBs, + SmallVectorImpl &Copies) { DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n'); DenseSet UsedByPhi; @@ -790,16 +790,13 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, // Update PredBB livein. RS->enterBasicBlock(PredBB); if (!PredBB->empty()) - RS->forward(prior(PredBB->end())); - BitVector RegsLiveAtExit(TRI->getNumRegs()); - RS->getRegsUsed(RegsLiveAtExit, false); - for (MachineBasicBlock::livein_iterator I = TailBB->livein_begin(), - E = TailBB->livein_end(); I != E; ++I) { - if (!RegsLiveAtExit[*I]) + RS->forward(std::prev(PredBB->end())); + for (unsigned LI : TailBB->liveins()) { + if (!RS->isRegUsed(LI, false)) // If a register is previously livein to the tail but it's not live // at the end of predecessor BB, then it should be added to its // livein list. - PredBB->addLiveIn(*I); + PredBB->addLiveIn(LI); } } @@ -840,7 +837,7 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, "TailDuplicate called on block with multiple successors!"); for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(), E = TailBB->succ_end(); I != E; ++I) - PredBB->addSuccessor(*I); + PredBB->addSuccessor(*I, MBPI->getEdgeWeight(TailBB, I)); Changed = true; ++NumTailDups; @@ -849,8 +846,8 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, // If TailBB was duplicated into all its predecessors except for the prior // block, which falls through unconditionally, move the contents of this // block into the prior block. - MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB)); - MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0; + MachineBasicBlock *PrevBB = std::prev(MachineFunction::iterator(TailBB)); + MachineBasicBlock *PriorTBB = nullptr, *PriorFBB = nullptr; SmallVector PriorCond; // This has to check PrevBB->succ_size() because EH edges are ignored by // AnalyzeBranch. @@ -955,8 +952,8 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, return Changed; } -/// RemoveDeadBlock - Remove the specified dead machine basic block from the -/// function, updating the CFG. +/// Remove the specified dead machine basic block from the function, updating +/// the CFG. void TailDuplicatePass::RemoveDeadBlock(MachineBasicBlock *MBB) { assert(MBB->pred_empty() && "MBB must be dead!"); DEBUG(dbgs() << "\nRemoving MBB: " << *MBB);