X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FMachineLICM.cpp;h=fa59b0c6aca24c4203599d0d4fb80142e343ce00;hb=bb300c512008269c1caabff9613a6dbc49f83fa4;hp=08678f81da51fb088f625f6aca242e69921754cb;hpb=92fca73d521758012e0e425eac33bb77e888112e;p=oota-llvm.git diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index 08678f81da5..fa59b0c6aca 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -10,17 +10,12 @@ // This pass performs loop invariant code motion on machine instructions. We // attempt to remove as much code from the body of a loop as possible. // -// This pass does not attempt to throttle itself to limit register pressure. -// The register allocation phases are expected to perform rematerialization -// to recover when register pressure is high. -// // This pass is not intended to be a replacement or a complete alternative // for the LLVM-IR-level LICM pass. It is only designed to hoist simple // constructs that are not exposed before lowering and instruction selection. // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "machine-licm" #include "llvm/CodeGen/Passes.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallSet.h" @@ -32,7 +27,7 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" -#include "llvm/MC/MCInstrItineraries.h" +#include "llvm/CodeGen/TargetSchedule.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -40,13 +35,27 @@ #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; +#define DEBUG_TYPE "machine-licm" + static cl::opt AvoidSpeculation("avoid-speculation", cl::desc("MachineLICM should avoid speculation"), cl::init(true), cl::Hidden); +static cl::opt +HoistCheapInsts("hoist-cheap-insts", + cl::desc("MachineLICM should hoist even cheap instructions"), + cl::init(false), cl::Hidden); + +static cl::opt +SinkInstsToAvoidSpills("sink-insts-to-avoid-spills", + cl::desc("MachineLICM should sink instructions into " + "loops to avoid register spills"), + cl::init(false), cl::Hidden); + STATISTIC(NumHoisted, "Number of machine instructions hoisted out of loops"); STATISTIC(NumLowRP, @@ -60,13 +69,12 @@ STATISTIC(NumPostRAHoisted, namespace { class MachineLICM : public MachineFunctionPass { - const TargetMachine *TM; const TargetInstrInfo *TII; const TargetLoweringBase *TLI; const TargetRegisterInfo *TRI; const MachineFrameInfo *MFI; MachineRegisterInfo *MRI; - const InstrItineraryData *InstrItins; + TargetSchedModel SchedModel; bool PreRegAlloc; // Various analyses that we use... @@ -92,7 +100,7 @@ namespace { SmallSet RegSeen; SmallVector RegPressure; - // Register pressure "limit" per register class. If the pressure + // Register pressure "limit" per register pressure set. If the pressure // is higher than the limit, then it's considered high. SmallVector RegLimit; @@ -130,7 +138,7 @@ namespace { void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); AU.addRequired(); - AU.addRequired(); + AU.addRequired(); AU.addPreserved(); AU.addPreserved(); MachineFunctionPass::getAnalysisUsage(AU); @@ -141,9 +149,6 @@ namespace { RegPressure.clear(); RegLimit.clear(); BackTrace.clear(); - for (DenseMap >::iterator - CI = CSEMap.begin(), CE = CSEMap.end(); CI != CE; ++CI) - CI->second.clear(); CSEMap.clear(); } @@ -205,7 +210,8 @@ namespace { /// CanCauseHighRegPressure - Visit BBs from header to current BB, /// check if hoisting an instruction of the given cost matrix can cause high /// register pressure. - bool CanCauseHighRegPressure(DenseMap &Cost, bool Cheap); + bool CanCauseHighRegPressure(const DenseMap &Cost, + bool Cheap); /// UpdateBackTraceRegPressure - Traverse the back trace from header to /// the current block and update their register pressures to reflect the @@ -240,21 +246,30 @@ namespace { void HoistOutOfLoop(MachineDomTreeNode *LoopHeaderNode); void HoistRegion(MachineDomTreeNode *N, bool IsHeader); - /// getRegisterClassIDAndCost - For a given MI, register, and the operand - /// index, return the ID and cost of its representative register class by - /// reference. - void getRegisterClassIDAndCost(const MachineInstr *MI, - unsigned Reg, unsigned OpIdx, - unsigned &RCId, unsigned &RCCost) const; + /// SinkIntoLoop - Sink instructions into loops if profitable. This + /// especially tries to prevent register spills caused by register pressure + /// if there is little to no overhead moving instructions into loops. + void SinkIntoLoop(); /// InitRegPressure - Find all virtual register references that are liveout /// of the preheader to initialize the starting "register pressure". Note /// this does not count live through (livein but not used) registers. void InitRegPressure(MachineBasicBlock *BB); + /// calcRegisterCost - Calculate the additional register pressure that the + /// registers used in MI cause. + /// + /// If 'ConsiderSeen' is true, updates 'RegSeen' and uses the information to + /// figure out which usages are live-ins. + /// FIXME: Figure out a way to consider 'RegSeen' from all code paths. + DenseMap calcRegisterCost(const MachineInstr *MI, + bool ConsiderSeen, + bool ConsiderUnseenAsDef); + /// UpdateRegPressure - Update estimate of register pressure after the /// specified instruction. - void UpdateRegPressure(const MachineInstr *MI); + void UpdateRegPressure(const MachineInstr *MI, + bool ConsiderUnseenAsDef = false); /// ExtractHoistableLoad - Unfold a load from the given machineinstr if /// the load itself could be hoisted. Return the unfolded and hoistable @@ -300,7 +315,7 @@ INITIALIZE_PASS_BEGIN(MachineLICM, "machinelicm", "Machine Loop Invariant Code Motion", false, false) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) -INITIALIZE_AG_DEPENDENCY(AliasAnalysis) +INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_END(MachineLICM, "machinelicm", "Machine Loop Invariant Code Motion", false, false) @@ -319,14 +334,17 @@ static bool LoopIsOuterMostWithPredecessor(MachineLoop *CurLoop) { } bool MachineLICM::runOnMachineFunction(MachineFunction &MF) { + if (skipOptnoneFunction(*MF.getFunction())) + return false; + Changed = FirstInLoop = false; - TM = &MF.getTarget(); - TII = TM->getInstrInfo(); - TLI = TM->getTargetLowering(); - TRI = TM->getRegisterInfo(); + const TargetSubtargetInfo &ST = MF.getSubtarget(); + TII = ST.getInstrInfo(); + TLI = ST.getTargetLowering(); + TRI = ST.getRegisterInfo(); MFI = MF.getFrameInfo(); MRI = &MF.getRegInfo(); - InstrItins = TM->getInstrItineraryData(); + SchedModel.init(ST.getSchedModel(), &ST, TII); PreRegAlloc = MRI->isSSA(); @@ -338,24 +356,23 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) { if (PreRegAlloc) { // Estimate register pressure during pre-regalloc pass. - unsigned NumRC = TRI->getNumRegClasses(); - RegPressure.resize(NumRC); + unsigned NumRPS = TRI->getNumRegPressureSets(); + RegPressure.resize(NumRPS); std::fill(RegPressure.begin(), RegPressure.end(), 0); - RegLimit.resize(NumRC); - for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(), - E = TRI->regclass_end(); I != E; ++I) - RegLimit[(*I)->getID()] = TRI->getRegPressureLimit(*I, MF); + RegLimit.resize(NumRPS); + for (unsigned i = 0, e = NumRPS; i != e; ++i) + RegLimit[i] = TRI->getRegPressureSetLimit(MF, i); } // Get our Loop information... MLI = &getAnalysis(); DT = &getAnalysis(); - AA = &getAnalysis(); + AA = &getAnalysis().getAAResults(); SmallVector Worklist(MLI->begin(), MLI->end()); while (!Worklist.empty()) { CurLoop = Worklist.pop_back_val(); - CurPreheader = 0; + CurPreheader = nullptr; ExitBlocks.clear(); // If this is done before regalloc, only visit outer-most preheader-sporting @@ -376,6 +393,9 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) { FirstInLoop = true; HoistOutOfLoop(N); CSEMap.clear(); + + if (SinkInstsToAvoidSpills) + SinkIntoLoop(); } } @@ -387,10 +407,10 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) { static bool InstructionStoresToFI(const MachineInstr *MI, int FI) { for (MachineInstr::mmo_iterator o = MI->memoperands_begin(), oe = MI->memoperands_end(); o != oe; ++o) { - if (!(*o)->isStore() || !(*o)->getValue()) + if (!(*o)->isStore() || !(*o)->getPseudoValue()) continue; if (const FixedStackPseudoSourceValue *Value = - dyn_cast((*o)->getValue())) { + dyn_cast((*o)->getPseudoValue())) { if (Value->getFrameIndex() == FI) return true; } @@ -509,15 +529,13 @@ void MachineLICM::HoistRegionPostRA() { // If the header of the loop containing this basic block is a landing pad, // then don't try to hoist instructions out of this loop. const MachineLoop *ML = MLI->getLoopFor(BB); - if (ML && ML->getHeader()->isLandingPad()) continue; + if (ML && ML->getHeader()->isEHPad()) continue; // Conservatively treat live-in's as an external def. // FIXME: That means a reload that're reused in successor block(s) will not // be LICM'ed. - for (MachineBasicBlock::livein_iterator I = BB->livein_begin(), - E = BB->livein_end(); I != E; ++I) { - unsigned Reg = *I; - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) + for (const auto &LI : BB->liveins()) { + for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI) PhysRegDefs.set(*AI); } @@ -688,6 +706,10 @@ void MachineLICM::ExitScopeIfDone(MachineDomTreeNode *Node, /// one pass without iteration. /// void MachineLICM::HoistOutOfLoop(MachineDomTreeNode *HeaderN) { + MachineBasicBlock *Preheader = getCurPreheader(); + if (!Preheader) + return; + SmallVector Scopes; SmallVector WorkList; DenseMap ParentMap; @@ -695,15 +717,15 @@ void MachineLICM::HoistOutOfLoop(MachineDomTreeNode *HeaderN) { // Perform a DFS walk to determine the order of visit. WorkList.push_back(HeaderN); - do { + while (!WorkList.empty()) { MachineDomTreeNode *Node = WorkList.pop_back_val(); - assert(Node != 0 && "Null dominator tree node?"); + assert(Node && "Null dominator tree node?"); MachineBasicBlock *BB = Node->getBlock(); // If the header of the loop containing this basic block is a landing pad, // then don't try to hoist instructions out of this loop. const MachineLoop *ML = MLI->getLoopFor(BB); - if (ML && ML->getHeader()->isLandingPad()) + if (ML && ML->getHeader()->isEHPad()) continue; // If this subregion is not in the top level loop at all, exit. @@ -729,28 +751,21 @@ void MachineLICM::HoistOutOfLoop(MachineDomTreeNode *HeaderN) { ParentMap[Child] = Node; WorkList.push_back(Child); } - } while (!WorkList.empty()); + } - if (Scopes.size() != 0) { - MachineBasicBlock *Preheader = getCurPreheader(); - if (!Preheader) - return; + if (Scopes.size() == 0) + return; - // Compute registers which are livein into the loop headers. - RegSeen.clear(); - BackTrace.clear(); - InitRegPressure(Preheader); - } + // Compute registers which are livein into the loop headers. + RegSeen.clear(); + BackTrace.clear(); + InitRegPressure(Preheader); // Now perform LICM. for (unsigned i = 0, e = Scopes.size(); i != e; ++i) { MachineDomTreeNode *Node = Scopes[i]; MachineBasicBlock *MBB = Node->getBlock(); - MachineBasicBlock *Preheader = getCurPreheader(); - if (!Preheader) - continue; - EnterScope(MBB); // Process the block @@ -769,27 +784,57 @@ void MachineLICM::HoistOutOfLoop(MachineDomTreeNode *HeaderN) { } } -static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) { - return MO.isKill() || MRI->hasOneNonDBGUse(MO.getReg()); -} +void MachineLICM::SinkIntoLoop() { + MachineBasicBlock *Preheader = getCurPreheader(); + if (!Preheader) + return; + + SmallVector Candidates; + for (MachineBasicBlock::instr_iterator I = Preheader->instr_begin(); + I != Preheader->instr_end(); ++I) { + // We need to ensure that we can safely move this instruction into the loop. + // As such, it must not have side-effects, e.g. such as a call has. + if (IsLoopInvariantInst(*I) && !HasLoopPHIUse(&*I)) + Candidates.push_back(&*I); + } -/// getRegisterClassIDAndCost - For a given MI, register, and the operand -/// index, return the ID and cost of its representative register class. -void -MachineLICM::getRegisterClassIDAndCost(const MachineInstr *MI, - unsigned Reg, unsigned OpIdx, - unsigned &RCId, unsigned &RCCost) const { - const TargetRegisterClass *RC = MRI->getRegClass(Reg); - MVT VT = *RC->vt_begin(); - if (VT == MVT::Untyped) { - RCId = RC->getID(); - RCCost = 1; - } else { - RCId = TLI->getRepRegClassFor(VT)->getID(); - RCCost = TLI->getRepRegClassCostFor(VT); + for (MachineInstr *I : Candidates) { + const MachineOperand &MO = I->getOperand(0); + if (!MO.isDef() || !MO.isReg() || !MO.getReg()) + continue; + if (!MRI->hasOneDef(MO.getReg())) + continue; + bool CanSink = true; + MachineBasicBlock *B = nullptr; + for (MachineInstr &MI : MRI->use_instructions(MO.getReg())) { + // FIXME: Come up with a proper cost model that estimates whether sinking + // the instruction (and thus possibly executing it on every loop + // iteration) is more expensive than a register. + // For now assumes that copies are cheap and thus almost always worth it. + if (!MI.isCopy()) { + CanSink = false; + break; + } + if (!B) { + B = MI.getParent(); + continue; + } + B = DT->findNearestCommonDominator(B, MI.getParent()); + if (!B) { + CanSink = false; + break; + } + } + if (!CanSink || !B || B == Preheader) + continue; + B->splice(B->getFirstNonPHI(), Preheader, I); } } +static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) { + return MO.isKill() || MRI->hasOneNonDBGUse(MO.getReg()); +} + /// InitRegPressure - Find all virtual register references that are liveout of /// the preheader to initialize the starting "register pressure". Note this /// does not count live through (livein but not used) registers. @@ -801,47 +846,36 @@ void MachineLICM::InitRegPressure(MachineBasicBlock *BB) { // defs as well. This happens whenever the preheader is created by splitting // the critical edge from the loop predecessor to the loop header. if (BB->pred_size() == 1) { - MachineBasicBlock *TBB = 0, *FBB = 0; + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; SmallVector Cond; if (!TII->AnalyzeBranch(*BB, TBB, FBB, Cond, false) && Cond.empty()) InitRegPressure(*BB->pred_begin()); } - for (MachineBasicBlock::iterator MII = BB->begin(), E = BB->end(); - MII != E; ++MII) { - MachineInstr *MI = &*MII; - for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || MO.isImplicit()) - continue; - unsigned Reg = MO.getReg(); - if (!TargetRegisterInfo::isVirtualRegister(Reg)) - continue; - - bool isNew = RegSeen.insert(Reg); - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (MO.isDef()) - RegPressure[RCId] += RCCost; - else { - bool isKill = isOperandKill(MO, MRI); - if (isNew && !isKill) - // Haven't seen this, it must be a livein. - RegPressure[RCId] += RCCost; - else if (!isNew && isKill) - RegPressure[RCId] -= RCCost; - } - } - } + for (const MachineInstr &MI : *BB) + UpdateRegPressure(&MI, /*ConsiderUnseenAsDef=*/true); } /// UpdateRegPressure - Update estimate of register pressure after the /// specified instruction. -void MachineLICM::UpdateRegPressure(const MachineInstr *MI) { - if (MI->isImplicitDef()) - return; +void MachineLICM::UpdateRegPressure(const MachineInstr *MI, + bool ConsiderUnseenAsDef) { + auto Cost = calcRegisterCost(MI, /*ConsiderSeen=*/true, ConsiderUnseenAsDef); + for (const auto &RPIdAndCost : Cost) { + unsigned Class = RPIdAndCost.first; + if (static_cast(RegPressure[Class]) < -RPIdAndCost.second) + RegPressure[Class] = 0; + else + RegPressure[Class] += RPIdAndCost.second; + } +} - SmallVector Defs; +DenseMap +MachineLICM::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen, + bool ConsiderUnseenAsDef) { + DenseMap Cost; + if (MI->isImplicitDef()) + return Cost; for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI->getOperand(i); if (!MO.isReg() || MO.isImplicit()) @@ -850,27 +884,33 @@ void MachineLICM::UpdateRegPressure(const MachineInstr *MI) { if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - bool isNew = RegSeen.insert(Reg); + // FIXME: It seems bad to use RegSeen only for some of these calculations. + bool isNew = ConsiderSeen ? RegSeen.insert(Reg).second : false; + const TargetRegisterClass *RC = MRI->getRegClass(Reg); + + RegClassWeight W = TRI->getRegClassWeight(RC); + int RCCost = 0; if (MO.isDef()) - Defs.push_back(Reg); - else if (!isNew && isOperandKill(MO, MRI)) { - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (RCCost > RegPressure[RCId]) - RegPressure[RCId] = 0; + RCCost = W.RegWeight; + else { + bool isKill = isOperandKill(MO, MRI); + if (isNew && !isKill && ConsiderUnseenAsDef) + // Haven't seen this, it must be a livein. + RCCost = W.RegWeight; + else if (!isNew && isKill) + RCCost = -W.RegWeight; + } + if (RCCost == 0) + continue; + const int *PS = TRI->getRegClassPressureSets(RC); + for (; *PS != -1; ++PS) { + if (Cost.find(*PS) == Cost.end()) + Cost[*PS] = RCCost; else - RegPressure[RCId] -= RCCost; + Cost[*PS] += RCCost; } } - - unsigned Idx = 0; - while (!Defs.empty()) { - unsigned Reg = Defs.pop_back_val(); - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, Idx, RCId, RCCost); - RegPressure[RCId] += RCCost; - ++Idx; - } + return Cost; } /// isLoadFromGOTOrConstantPool - Return true if this machine instruction @@ -879,10 +919,9 @@ static bool isLoadFromGOTOrConstantPool(MachineInstr &MI) { assert (MI.mayLoad() && "Expected MI that loads!"); for (MachineInstr::mmo_iterator I = MI.memoperands_begin(), E = MI.memoperands_end(); I != E; ++I) { - if (const Value *V = (*I)->getValue()) { - if (const PseudoSourceValue *PSV = dyn_cast(V)) - if (PSV == PSV->getGOT() || PSV == PSV->getConstantPool()) - return true; + if (const PseudoSourceValue *PSV = (*I)->getPseudoValue()) { + if (PSV->isGOT() || PSV->isConstantPool()) + return true; } } return false; @@ -894,7 +933,7 @@ static bool isLoadFromGOTOrConstantPool(MachineInstr &MI) { bool MachineLICM::IsLICMCandidate(MachineInstr &I) { // Check if it's safe to move the instruction. bool DontMoveAcrossStore = true; - if (!I.isSafeToMove(TII, AA, DontMoveAcrossStore)) + if (!I.isSafeToMove(AA, DontMoveAcrossStore)) return false; // If it is load then check if it is guaranteed to execute by making sure that @@ -972,10 +1011,10 @@ bool MachineLICM::HasLoopPHIUse(const MachineInstr *MI) const { SmallVector Work(1, MI); do { MI = Work.pop_back_val(); - for (ConstMIOperands MO(MI); MO.isValid(); ++MO) { - if (!MO->isReg() || !MO->isDef()) + for (const MachineOperand &MO : MI->operands()) { + if (!MO.isReg() || !MO.isDef()) continue; - unsigned Reg = MO->getReg(); + unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; for (MachineInstr &UseMI : MRI->use_instructions(Reg)) { @@ -1006,7 +1045,7 @@ bool MachineLICM::HasLoopPHIUse(const MachineInstr *MI) const { /// it 'high'. bool MachineLICM::HasHighOperandLatency(MachineInstr &MI, unsigned DefIdx, unsigned Reg) const { - if (!InstrItins || InstrItins->isEmpty() || MRI->use_nodbg_empty(Reg)) + if (MRI->use_nodbg_empty(Reg)) return false; for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) { @@ -1022,7 +1061,7 @@ bool MachineLICM::HasHighOperandLatency(MachineInstr &MI, if (MOReg != Reg) continue; - if (TII->hasHighOperandLatency(InstrItins, MRI, &MI, DefIdx, &UseMI, i)) + if (TII->hasHighOperandLatency(SchedModel, MRI, &MI, DefIdx, &UseMI, i)) return true; } @@ -1036,10 +1075,8 @@ bool MachineLICM::HasHighOperandLatency(MachineInstr &MI, /// IsCheapInstruction - Return true if the instruction is marked "cheap" or /// the operand latency between its def and a use is one or less. bool MachineLICM::IsCheapInstruction(MachineInstr &MI) const { - if (MI.isAsCheapAsAMove() || MI.isCopyLike()) + if (TII->isAsCheapAsAMove(&MI) || MI.isCopyLike()) return true; - if (!InstrItins || InstrItins->isEmpty()) - return false; bool isCheap = false; unsigned NumDefs = MI.getDesc().getNumDefs(); @@ -1052,7 +1089,7 @@ bool MachineLICM::IsCheapInstruction(MachineInstr &MI) const { if (TargetRegisterInfo::isPhysicalRegister(Reg)) continue; - if (!TII->hasLowDefLatency(InstrItins, &MI, i)) + if (!TII->hasLowDefLatency(SchedModel, &MI, i)) return false; isCheap = true; } @@ -1063,27 +1100,23 @@ bool MachineLICM::IsCheapInstruction(MachineInstr &MI) const { /// CanCauseHighRegPressure - Visit BBs from header to current BB, check /// if hoisting an instruction of the given cost matrix can cause high /// register pressure. -bool MachineLICM::CanCauseHighRegPressure(DenseMap &Cost, +bool MachineLICM::CanCauseHighRegPressure(const DenseMap& Cost, bool CheapInstr) { - for (DenseMap::iterator CI = Cost.begin(), CE = Cost.end(); - CI != CE; ++CI) { - if (CI->second <= 0) + for (const auto &RPIdAndCost : Cost) { + if (RPIdAndCost.second <= 0) continue; - unsigned RCId = CI->first; - unsigned Limit = RegLimit[RCId]; - int Cost = CI->second; + unsigned Class = RPIdAndCost.first; + int Limit = RegLimit[Class]; // Don't hoist cheap instructions if they would increase register pressure, // even if we're under the limit. - if (CheapInstr) + if (CheapInstr && !HoistCheapInsts) return true; - for (unsigned i = BackTrace.size(); i != 0; --i) { - SmallVectorImpl &RP = BackTrace[i-1]; - if (RP[RCId] + Cost >= Limit) + for (const auto &RP : BackTrace) + if (static_cast(RP[Class]) + RPIdAndCost.second >= Limit) return true; - } } return false; @@ -1093,46 +1126,15 @@ bool MachineLICM::CanCauseHighRegPressure(DenseMap &Cost, /// current block and update their register pressures to reflect the effect /// of hoisting MI from the current block to the preheader. void MachineLICM::UpdateBackTraceRegPressure(const MachineInstr *MI) { - if (MI->isImplicitDef()) - return; - // First compute the 'cost' of the instruction, i.e. its contribution // to register pressure. - DenseMap Cost; - for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || MO.isImplicit()) - continue; - unsigned Reg = MO.getReg(); - if (!TargetRegisterInfo::isVirtualRegister(Reg)) - continue; - - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (MO.isDef()) { - DenseMap::iterator CI = Cost.find(RCId); - if (CI != Cost.end()) - CI->second += RCCost; - else - Cost.insert(std::make_pair(RCId, RCCost)); - } else if (isOperandKill(MO, MRI)) { - DenseMap::iterator CI = Cost.find(RCId); - if (CI != Cost.end()) - CI->second -= RCCost; - else - Cost.insert(std::make_pair(RCId, -RCCost)); - } - } + auto Cost = calcRegisterCost(MI, /*ConsiderSeen=*/false, + /*ConsiderUnseenAsDef=*/false); // Update register pressure of blocks from loop header to current block. - for (unsigned i = 0, e = BackTrace.size(); i != e; ++i) { - SmallVectorImpl &RP = BackTrace[i]; - for (DenseMap::iterator CI = Cost.begin(), CE = Cost.end(); - CI != CE; ++CI) { - unsigned RCId = CI->first; - RP[RCId] += CI->second; - } - } + for (auto &RP : BackTrace) + for (const auto &RPIdAndCost : Cost) + RP[RPIdAndCost.first] += RPIdAndCost.second; } /// IsProfitableToHoist - Return true if it is potentially profitable to hoist @@ -1167,15 +1169,8 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { if (TII->isTriviallyReMaterializable(&MI, AA)) return true; - // Estimate register pressure to determine whether to LICM the instruction. - // In low register pressure situation, we can be more aggressive about - // hoisting. Also, favors hoisting long latency instructions even in - // moderately high pressure situation. - // Cheap instructions will only be hoisted if they don't increase register - // pressure at all. // FIXME: If there are long latency loop-invariant instructions inside the // loop at this point, why didn't the optimizer's LICM hoist them? - DenseMap Cost; for (unsigned i = 0, e = MI.getDesc().getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI.getOperand(i); if (!MO.isReg() || MO.isImplicit()) @@ -1183,24 +1178,22 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - - unsigned RCId, RCCost; - getRegisterClassIDAndCost(&MI, Reg, i, RCId, RCCost); - if (MO.isDef()) { - if (HasHighOperandLatency(MI, i, Reg)) { - DEBUG(dbgs() << "Hoist High Latency: " << MI); - ++NumHighLatency; - return true; - } - Cost[RCId] += RCCost; - } else if (isOperandKill(MO, MRI)) { - // Is a virtual register use is a kill, hoisting it out of the loop - // may actually reduce register pressure or be register pressure - // neutral. - Cost[RCId] -= RCCost; + if (MO.isDef() && HasHighOperandLatency(MI, i, Reg)) { + DEBUG(dbgs() << "Hoist High Latency: " << MI); + ++NumHighLatency; + return true; } } + // Estimate register pressure to determine whether to LICM the instruction. + // In low register pressure situation, we can be more aggressive about + // hoisting. Also, favors hoisting long latency instructions even in + // moderately high pressure situation. + // Cheap instructions will only be hoisted if they don't increase register + // pressure at all. + auto Cost = calcRegisterCost(&MI, /*ConsiderSeen=*/false, + /*ConsiderUnseenAsDef=*/false); + // Visit BBs from header to current BB, if hoisting this doesn't cause // high register pressure, then it's safe to proceed. if (!CanCauseHighRegPressure(Cost, CheapInstr)) { @@ -1238,13 +1231,13 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { MachineInstr *MachineLICM::ExtractHoistableLoad(MachineInstr *MI) { // Don't unfold simple loads. if (MI->canFoldAsLoad()) - return 0; + return nullptr; // If not, we may be able to unfold a load and hoist that. // First test whether the instruction is loading from an amenable // memory location. if (!MI->isInvariantLoad(AA)) - return 0; + return nullptr; // Next determine the register class for a temporary register. unsigned LoadRegIndex; @@ -1253,9 +1246,9 @@ MachineInstr *MachineLICM::ExtractHoistableLoad(MachineInstr *MI) { /*UnfoldLoad=*/true, /*UnfoldStore=*/false, &LoadRegIndex); - if (NewOpc == 0) return 0; + if (NewOpc == 0) return nullptr; const MCInstrDesc &MID = TII->get(NewOpc); - if (MID.getNumDefs() != 1) return 0; + if (MID.getNumDefs() != 1) return nullptr; MachineFunction &MF = *MI->getParent()->getParent(); const TargetRegisterClass *RC = TII->getRegClass(MID, LoadRegIndex, TRI, MF); // Ok, we're unfolding. Create a temporary register and do the unfold. @@ -1281,7 +1274,7 @@ MachineInstr *MachineLICM::ExtractHoistableLoad(MachineInstr *MI) { if (!IsLoopInvariantInst(*NewMIs[0]) || !IsProfitableToHoist(*NewMIs[0])) { NewMIs[0]->eraseFromParent(); NewMIs[1]->eraseFromParent(); - return 0; + return nullptr; } // Update register pressure for the unfolded instruction. @@ -1296,15 +1289,7 @@ void MachineLICM::InitCSEMap(MachineBasicBlock *BB) { for (MachineBasicBlock::iterator I = BB->begin(),E = BB->end(); I != E; ++I) { const MachineInstr *MI = &*I; unsigned Opcode = MI->getOpcode(); - DenseMap >::iterator - CI = CSEMap.find(Opcode); - if (CI != CSEMap.end()) - CI->second.push_back(MI); - else { - std::vector CSEMIs; - CSEMIs.push_back(MI); - CSEMap.insert(std::make_pair(Opcode, CSEMIs)); - } + CSEMap[Opcode].push_back(MI); } } @@ -1313,10 +1298,10 @@ MachineLICM::LookForDuplicate(const MachineInstr *MI, std::vector &PrevMIs) { for (unsigned i = 0, e = PrevMIs.size(); i != e; ++i) { const MachineInstr *PrevMI = PrevMIs[i]; - if (TII->produceSameValue(MI, PrevMI, (PreRegAlloc ? MRI : 0))) + if (TII->produceSameValue(MI, PrevMI, (PreRegAlloc ? MRI : nullptr))) return PrevMI; } - return 0; + return nullptr; } bool MachineLICM::EliminateCSE(MachineInstr *MI, @@ -1387,7 +1372,7 @@ bool MachineLICM::MayCSE(MachineInstr *MI) { if (CI == CSEMap.end() || MI->isImplicitDef()) return false; - return LookForDuplicate(MI, CI->second) != 0; + return LookForDuplicate(MI, CI->second) != nullptr; } /// Hoist - When an instruction is found to use only loop invariant operands @@ -1444,11 +1429,8 @@ bool MachineLICM::Hoist(MachineInstr *MI, MachineBasicBlock *Preheader) { // Add to the CSE map. if (CI != CSEMap.end()) CI->second.push_back(MI); - else { - std::vector CSEMIs; - CSEMIs.push_back(MI); - CSEMap.insert(std::make_pair(Opcode, CSEMIs)); - } + else + CSEMap[Opcode].push_back(MI); } ++NumHoisted; @@ -1463,7 +1445,7 @@ MachineBasicBlock *MachineLICM::getCurPreheader() { // If we've tried to get a preheader and failed, don't try again. if (CurPreheader == reinterpret_cast(-1)) - return 0; + return nullptr; if (!CurPreheader) { CurPreheader = CurLoop->getLoopPreheader(); @@ -1471,13 +1453,13 @@ MachineBasicBlock *MachineLICM::getCurPreheader() { MachineBasicBlock *Pred = CurLoop->getLoopPredecessor(); if (!Pred) { CurPreheader = reinterpret_cast(-1); - return 0; + return nullptr; } CurPreheader = Pred->SplitCriticalEdge(CurLoop->getHeader(), this); if (!CurPreheader) { CurPreheader = reinterpret_cast(-1); - return 0; + return nullptr; } } }