X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FLiveRangeCalc.cpp;h=c408615d42e28afbe1b7574689b0ac8911eb4a37;hb=7a36028a83819aa49568d5d852a0f8fdae62aafb;hp=eedf924388e1196350eafdbe7c4b6ee77446b93f;hpb=edd4f8ba4bed5b999c6a726b7991241cf1840350;p=oota-llvm.git diff --git a/lib/CodeGen/LiveRangeCalc.cpp b/lib/CodeGen/LiveRangeCalc.cpp index eedf924388e..c408615d42e 100644 --- a/lib/CodeGen/LiveRangeCalc.cpp +++ b/lib/CodeGen/LiveRangeCalc.cpp @@ -11,117 +11,293 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "regalloc" #include "LiveRangeCalc.h" #include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" using namespace llvm; -void LiveRangeCalc::reset(const MachineFunction *MF) { - unsigned N = MF->getNumBlockIDs(); +#define DEBUG_TYPE "regalloc" + +void LiveRangeCalc::resetLiveOutMap() { + unsigned NumBlocks = MF->getNumBlockIDs(); Seen.clear(); - Seen.resize(N); - LiveOut.resize(N); + Seen.resize(NumBlocks); + Map.resize(NumBlocks); +} + +void LiveRangeCalc::reset(const MachineFunction *mf, + SlotIndexes *SI, + MachineDominatorTree *MDT, + VNInfo::Allocator *VNIA) { + MF = mf; + MRI = &MF->getRegInfo(); + Indexes = SI; + DomTree = MDT; + Alloc = VNIA; + resetLiveOutMap(); LiveIn.clear(); } -// Transfer information from the LiveIn vector to the live ranges. -void LiveRangeCalc::updateLiveIns(VNInfo *OverrideVNI, SlotIndexes *Indexes) { - for (SmallVectorImpl::iterator I = LiveIn.begin(), - E = LiveIn.end(); I != E; ++I) { - if (!I->DomNode) +static void createDeadDef(SlotIndexes &Indexes, VNInfo::Allocator &Alloc, + LiveRange &LR, const MachineOperand &MO) { + const MachineInstr *MI = MO.getParent(); + SlotIndex DefIdx = + Indexes.getInstructionIndex(MI).getRegSlot(MO.isEarlyClobber()); + + // Create the def in LR. This may find an existing def. + LR.createDeadDef(DefIdx, Alloc); +} + +void LiveRangeCalc::calculate(LiveInterval &LI, bool TrackSubRegs) { + assert(MRI && Indexes && "call reset() first"); + + // Step 1: Create minimal live segments for every definition of Reg. + // Visit all def operands. If the same instruction has multiple defs of Reg, + // createDeadDef() will deduplicate. + const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo(); + unsigned Reg = LI.reg; + for (const MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) { + if (!MO.isDef() && !MO.readsReg()) continue; - MachineBasicBlock *MBB = I->DomNode->getBlock(); - VNInfo *VNI = OverrideVNI ? OverrideVNI : I->Value; - assert(VNI && "No live-in value found"); + unsigned SubReg = MO.getSubReg(); + if (LI.hasSubRanges() || (SubReg != 0 && TrackSubRegs)) { + LaneBitmask Mask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg) + : MRI->getMaxLaneMaskForVReg(Reg); + + // If this is the first time we see a subregister def, initialize + // subranges by creating a copy of the main range. + if (!LI.hasSubRanges() && !LI.empty()) { + LaneBitmask ClassMask = MRI->getMaxLaneMaskForVReg(Reg); + LI.createSubRangeFrom(*Alloc, ClassMask, LI); + } + for (LiveInterval::SubRange &S : LI.subranges()) { + // A Mask for subregs common to the existing subrange and current def. + LaneBitmask Common = S.LaneMask & Mask; + if (Common == 0) + continue; + // A Mask for subregs covered by the subrange but not the current def. + LaneBitmask LRest = S.LaneMask & ~Mask; + LiveInterval::SubRange *CommonRange; + if (LRest != 0) { + // Split current subrange into Common and LRest ranges. + S.LaneMask = LRest; + CommonRange = LI.createSubRangeFrom(*Alloc, Common, S); + } else { + assert(Common == S.LaneMask); + CommonRange = &S; + } + if (MO.isDef()) + createDeadDef(*Indexes, *Alloc, *CommonRange, MO); + Mask &= ~Common; + } + // Create a new SubRange for subregs we did not cover yet. + if (Mask != 0) { + LiveInterval::SubRange *NewRange = LI.createSubRange(*Alloc, Mask); + if (MO.isDef()) + createDeadDef(*Indexes, *Alloc, *NewRange, MO); + } + } + + // Create the def in the main liverange. We do not have to do this if + // subranges are tracked as we recreate the main range later in this case. + if (MO.isDef() && !LI.hasSubRanges()) + createDeadDef(*Indexes, *Alloc, LI, MO); + } + + // We may have created empty live ranges for partially undefined uses, we + // can't keep them because we won't find defs in them later. + LI.removeEmptySubRanges(); + + // Step 2: Extend live segments to all uses, constructing SSA form as + // necessary. + if (LI.hasSubRanges()) { + for (LiveInterval::SubRange &S : LI.subranges()) { + resetLiveOutMap(); + extendToUses(S, Reg, S.LaneMask); + } + LI.clear(); + LI.constructMainRangeFromSubranges(*Indexes, *Alloc); + } else { + resetLiveOutMap(); + extendToUses(LI, Reg, ~0u); + } +} + + +void LiveRangeCalc::createDeadDefs(LiveRange &LR, unsigned Reg) { + assert(MRI && Indexes && "call reset() first"); + + // Visit all def operands. If the same instruction has multiple defs of Reg, + // LR.createDeadDef() will deduplicate. + for (MachineOperand &MO : MRI->def_operands(Reg)) + createDeadDef(*Indexes, *Alloc, LR, MO); +} + + +void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg, + LaneBitmask Mask) { + // Visit all operands that read Reg. This may include partial defs. + const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo(); + for (MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) { + // Clear all kill flags. They will be reinserted after register allocation + // by LiveIntervalAnalysis::addKillFlags(). + if (MO.isUse()) + MO.setIsKill(false); + else { + // We only care about uses, but on the main range (mask ~0u) this includes + // the "virtual" reads happening for subregister defs. + if (Mask != ~0u) + continue; + } + + if (!MO.readsReg()) + continue; + unsigned SubReg = MO.getSubReg(); + if (SubReg != 0) { + LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(SubReg); + // Ignore uses not covering the current subrange. + if ((SubRegMask & Mask) == 0) + continue; + } + + // Determine the actual place of the use. + const MachineInstr *MI = MO.getParent(); + unsigned OpNo = (&MO - &MI->getOperand(0)); + SlotIndex UseIdx; + if (MI->isPHI()) { + assert(!MO.isDef() && "Cannot handle PHI def of partial register."); + // The actual place where a phi operand is used is the end of the pred + // MBB. PHI operands are paired: (Reg, PredMBB). + UseIdx = Indexes->getMBBEndIdx(MI->getOperand(OpNo+1).getMBB()); + } else { + // Check for early-clobber redefs. + bool isEarlyClobber = false; + unsigned DefIdx; + if (MO.isDef()) + isEarlyClobber = MO.isEarlyClobber(); + else if (MI->isRegTiedToDefOperand(OpNo, &DefIdx)) { + // FIXME: This would be a lot easier if tied early-clobber uses also + // had an early-clobber flag. + isEarlyClobber = MI->getOperand(DefIdx).isEarlyClobber(); + } + UseIdx = Indexes->getInstructionIndex(MI).getRegSlot(isEarlyClobber); + } + + // MI is reading Reg. We may have visited MI before if it happens to be + // reading Reg multiple times. That is OK, extend() is idempotent. + extend(LR, UseIdx, Reg); + } +} + + +void LiveRangeCalc::updateFromLiveIns() { + LiveRangeUpdater Updater; + for (const LiveInBlock &I : LiveIn) { + if (!I.DomNode) + continue; + MachineBasicBlock *MBB = I.DomNode->getBlock(); + assert(I.Value && "No live-in value found"); SlotIndex Start, End; - tie(Start, End) = Indexes->getMBBRange(MBB); + std::tie(Start, End) = Indexes->getMBBRange(MBB); - if (I->Kill.isValid()) - I->LI->addRange(LiveRange(Start, I->Kill, VNI)); + if (I.Kill.isValid()) + // Value is killed inside this block. + End = I.Kill; else { - I->LI->addRange(LiveRange(Start, End, VNI)); - // The value is live-through, update LiveOut as well. Defer the Domtree - // lookup until it is needed. + // The value is live-through, update LiveOut as well. + // Defer the Domtree lookup until it is needed. assert(Seen.test(MBB->getNumber())); - LiveOut[MBB] = LiveOutPair(VNI, (MachineDomTreeNode *)0); + Map[MBB] = LiveOutPair(I.Value, nullptr); } + Updater.setDest(&I.LR); + Updater.add(Start, End, I.Value); } LiveIn.clear(); } -void LiveRangeCalc::extend(LiveInterval *LI, - SlotIndex Kill, - SlotIndexes *Indexes, - MachineDominatorTree *DomTree, - VNInfo::Allocator *Alloc) { - assert(LI && "Missing live range"); - assert(Kill.isValid() && "Invalid SlotIndex"); +void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg) { + assert(Use.isValid() && "Invalid SlotIndex"); assert(Indexes && "Missing SlotIndexes"); assert(DomTree && "Missing dominator tree"); - SlotIndex LastUse = Kill.getPrevSlot(); - MachineBasicBlock *KillMBB = Indexes->getMBBFromIndex(LastUse); - assert(Kill && "No MBB at Kill"); + MachineBasicBlock *UseMBB = Indexes->getMBBFromIndex(Use.getPrevSlot()); + assert(UseMBB && "No MBB at Use"); // Is there a def in the same MBB we can extend? - if (LI->extendInBlock(Indexes->getMBBStartIdx(KillMBB), LastUse)) + if (LR.extendInBlock(Indexes->getMBBStartIdx(UseMBB), Use)) return; - // Find the single reaching def, or determine if Kill is jointly dominated by + // Find the single reaching def, or determine if Use is jointly dominated by // multiple values, and we may need to create even more phi-defs to preserve // VNInfo SSA form. Perform a search for all predecessor blocks where we // know the dominating VNInfo. - VNInfo *VNI = findReachingDefs(LI, KillMBB, Kill, Indexes, DomTree); + if (findReachingDefs(LR, *UseMBB, Use, PhysReg)) + return; // When there were multiple different values, we may need new PHIs. - if (!VNI) - updateSSA(Indexes, DomTree, Alloc); - - updateLiveIns(VNI, Indexes); + calculateValues(); } // This function is called by a client after using the low-level API to add // live-out and live-in blocks. The unique value optimization is not // available, SplitEditor::transferValues handles that case directly anyway. -void LiveRangeCalc::calculateValues(SlotIndexes *Indexes, - MachineDominatorTree *DomTree, - VNInfo::Allocator *Alloc) { +void LiveRangeCalc::calculateValues() { assert(Indexes && "Missing SlotIndexes"); assert(DomTree && "Missing dominator tree"); - updateSSA(Indexes, DomTree, Alloc); - updateLiveIns(0, Indexes); + updateSSA(); + updateFromLiveIns(); } -VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, - MachineBasicBlock *KillMBB, - SlotIndex Kill, - SlotIndexes *Indexes, - MachineDominatorTree *DomTree) { - // Blocks where LI should be live-in. - SmallVector WorkList(1, KillMBB); +bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB, + SlotIndex Use, unsigned PhysReg) { + unsigned UseMBBNum = UseMBB.getNumber(); + + // Block numbers where LR should be live-in. + SmallVector WorkList(1, UseMBBNum); // Remember if we have seen more than one value. bool UniqueVNI = true; - VNInfo *TheVNI = 0; + VNInfo *TheVNI = nullptr; // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { - MachineBasicBlock *MBB = WorkList[i]; - assert(!MBB->pred_empty() && "Value live-in to entry block?"); + MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]); + +#ifndef NDEBUG + if (MBB->pred_empty()) { + MBB->getParent()->verify(); + errs() << "Use of " << PrintReg(PhysReg) + << " does not have a corresponding definition on every path:\n"; + const MachineInstr *MI = Indexes->getInstructionFromIndex(Use); + if (MI != nullptr) + errs() << Use << " " << *MI; + llvm_unreachable("Use not jointly dominated by defs."); + } + + if (TargetRegisterInfo::isPhysicalRegister(PhysReg) && + !MBB->isLiveIn(PhysReg)) { + MBB->getParent()->verify(); + errs() << "The register " << PrintReg(PhysReg) + << " needs to be live in to BB#" << MBB->getNumber() + << ", but is missing from the live-in list.\n"; + llvm_unreachable("Invalid global physical register"); + } +#endif + for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), - PE = MBB->pred_end(); PI != PE; ++PI) { + PE = MBB->pred_end(); PI != PE; ++PI) { MachineBasicBlock *Pred = *PI; // Is this a known live-out block? if (Seen.test(Pred->getNumber())) { - if (VNInfo *VNI = LiveOut[Pred].first) { + if (VNInfo *VNI = Map[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; @@ -130,11 +306,11 @@ VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, } SlotIndex Start, End; - tie(Start, End) = Indexes->getMBBRange(Pred); + std::tie(Start, End) = Indexes->getMBBRange(Pred); // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. - VNInfo *VNI = LI->extendInBlock(Start, End.getPrevSlot()); + VNInfo *VNI = LR.extendInBlock(Start, End); setLiveOutValue(Pred, VNI); if (VNI) { if (TheVNI && TheVNI != VNI) @@ -144,34 +320,56 @@ VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, } // No, we need a live-in value for Pred as well - if (Pred != KillMBB) - WorkList.push_back(Pred); + if (Pred != &UseMBB) + WorkList.push_back(Pred->getNumber()); else - // Loopback to KillMBB, so value is really live through. - Kill = SlotIndex(); + // Loopback to UseMBB, so value is really live through. + Use = SlotIndex(); } } - // Transfer WorkList to LiveInBlocks in reverse order. - // This ordering works best with updateSSA(). LiveIn.clear(); - LiveIn.reserve(WorkList.size()); - while(!WorkList.empty()) - addLiveInBlock(LI, DomTree->getNode(WorkList.pop_back_val())); - // The kill block may not be live-through. - assert(LiveIn.back().DomNode->getBlock() == KillMBB); - LiveIn.back().Kill = Kill; + // Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but + // neither require it. Skip the sorting overhead for small updates. + if (WorkList.size() > 4) + array_pod_sort(WorkList.begin(), WorkList.end()); + + // If a unique reaching def was found, blit in the live ranges immediately. + if (UniqueVNI) { + LiveRangeUpdater Updater(&LR); + for (SmallVectorImpl::const_iterator I = WorkList.begin(), + E = WorkList.end(); I != E; ++I) { + SlotIndex Start, End; + std::tie(Start, End) = Indexes->getMBBRange(*I); + // Trim the live range in UseMBB. + if (*I == UseMBBNum && Use.isValid()) + End = Use; + else + Map[MF->getBlockNumbered(*I)] = LiveOutPair(TheVNI, nullptr); + Updater.add(Start, End, TheVNI); + } + return true; + } + + // Multiple values were found, so transfer the work list to the LiveIn array + // where UpdateSSA will use it as a work list. + LiveIn.reserve(WorkList.size()); + for (SmallVectorImpl::const_iterator + I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { + MachineBasicBlock *MBB = MF->getBlockNumbered(*I); + addLiveInBlock(LR, DomTree->getNode(MBB)); + if (MBB == &UseMBB) + LiveIn.back().Kill = Use; + } - return UniqueVNI ? TheVNI : 0; + return false; } // This is essentially the same iterative algorithm that SSAUpdater uses, // except we already have a dominator tree, so we don't have to recompute it. -void LiveRangeCalc::updateSSA(SlotIndexes *Indexes, - MachineDominatorTree *DomTree, - VNInfo::Allocator *Alloc) { +void LiveRangeCalc::updateSSA() { assert(Indexes && "Missing SlotIndexes"); assert(DomTree && "Missing dominator tree"); @@ -181,9 +379,8 @@ void LiveRangeCalc::updateSSA(SlotIndexes *Indexes, Changes = 0; // Propagate live-out values down the dominator tree, inserting phi-defs // when necessary. - for (SmallVectorImpl::iterator I = LiveIn.begin(), - E = LiveIn.end(); I != E; ++I) { - MachineDomTreeNode *Node = I->DomNode; + for (LiveInBlock &I : LiveIn) { + MachineDomTreeNode *Node = I.DomNode; // Skip block if the live-in value has already been determined. if (!Node) continue; @@ -199,16 +396,16 @@ void LiveRangeCalc::updateSSA(SlotIndexes *Indexes, // immediate dominator. Check if any of them have live-out values that are // properly dominated by IDom. If so, we need a phi-def here. if (!needPHI) { - IDomValue = LiveOut[IDom->getBlock()]; + IDomValue = Map[IDom->getBlock()]; // Cache the DomTree node that defined the value. if (IDomValue.first && !IDomValue.second) - LiveOut[IDom->getBlock()].second = IDomValue.second = + Map[IDom->getBlock()].second = IDomValue.second = DomTree->getNode(Indexes->getMBBFromIndex(IDomValue.first->def)); for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { - LiveOutPair &Value = LiveOut[*PI]; + LiveOutPair &Value = Map[*PI]; if (!Value.first || Value.first == IDomValue.first) continue; @@ -230,33 +427,33 @@ void LiveRangeCalc::updateSSA(SlotIndexes *Indexes, // The value may be live-through even if Kill is set, as can happen when // we are called from extendRange. In that case LiveOutSeen is true, and // LiveOut indicates a foreign or missing value. - LiveOutPair &LOP = LiveOut[MBB]; + LiveOutPair &LOP = Map[MBB]; // Create a phi-def if required. if (needPHI) { ++Changes; assert(Alloc && "Need VNInfo allocator to create PHI-defs"); SlotIndex Start, End; - tie(Start, End) = Indexes->getMBBRange(MBB); - VNInfo *VNI = I->LI->getNextValue(Start, 0, *Alloc); - VNI->setIsPHIDef(true); - I->Value = VNI; + std::tie(Start, End) = Indexes->getMBBRange(MBB); + LiveRange &LR = I.LR; + VNInfo *VNI = LR.getNextValue(Start, *Alloc); + I.Value = VNI; // This block is done, we know the final value. - I->DomNode = 0; + I.DomNode = nullptr; - // Add liveness since updateLiveIns now skips this node. - if (I->Kill.isValid()) - I->LI->addRange(LiveRange(Start, I->Kill, VNI)); + // Add liveness since updateFromLiveIns now skips this node. + if (I.Kill.isValid()) + LR.addSegment(LiveInterval::Segment(Start, I.Kill, VNI)); else { - I->LI->addRange(LiveRange(Start, End, VNI)); + LR.addSegment(LiveInterval::Segment(Start, End, VNI)); LOP = LiveOutPair(VNI, Node); } } else if (IDomValue.first) { // No phi-def here. Remember incoming value. - I->Value = IDomValue.first; + I.Value = IDomValue.first; // If the IDomValue is killed in the block, don't propagate through. - if (I->Kill.isValid()) + if (I.Kill.isValid()) continue; // Propagate IDomValue if it isn't killed: