//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "regalloc"
+#include "llvm/CodeGen/Passes.h"
#include "AllocationOrder.h"
#include "InterferenceCache.h"
#include "LiveDebugVariables.h"
#include "RegAllocBase.h"
-#include "Spiller.h"
#include "SpillPlacement.h"
+#include "Spiller.h"
#include "SplitKit.h"
-#include "VirtRegMap.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Function.h"
-#include "llvm/PassAnalysisSupport.h"
#include "llvm/CodeGen/CalcSpillWeights.h"
#include "llvm/CodeGen/EdgeBundles.h"
#include "llvm/CodeGen/LiveIntervalAnalysis.h"
#include "llvm/CodeGen/LiveRangeEdit.h"
+#include "llvm/CodeGen/LiveRegMatrix.h"
#include "llvm/CodeGen/LiveStackAnalysis.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/RegAllocRegistry.h"
-#include "llvm/Target/TargetOptions.h"
+#include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/PassAnalysisSupport.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/Timer.h"
-
+#include "llvm/Support/raw_ostream.h"
#include <queue>
using namespace llvm;
// analyses
SlotIndexes *Indexes;
- LiveStacks *LS;
+ MachineBlockFrequencyInfo *MBFI;
MachineDominatorTree *DomTree;
MachineLoopInfo *Loops;
EdgeBundles *Bundles;
LiveDebugVariables *DebugVars;
// state
- std::auto_ptr<Spiller> SpillerInstance;
+ OwningPtr<Spiller> SpillerInstance;
std::priority_queue<std::pair<unsigned, unsigned> > Queue;
unsigned NextCascade;
RS_Done
};
+#ifndef NDEBUG
static const char *const StageName[];
+#endif
// RegInfo - Keep additional information about each live range.
struct RegInfo {
void setStage(Iterator Begin, Iterator End, LiveRangeStage NewStage) {
ExtraRegInfo.resize(MRI->getNumVirtRegs());
for (;Begin != End; ++Begin) {
- unsigned Reg = (*Begin)->reg;
+ unsigned Reg = *Begin;
if (ExtraRegInfo[Reg].Stage == RS_New)
ExtraRegInfo[Reg].Stage = NewStage;
}
EvictionCost(unsigned B = 0) : BrokenHints(B), MaxWeight(0) {}
+ bool isMax() const { return BrokenHints == ~0u; }
+
bool operator<(const EvictionCost &O) const {
if (BrokenHints != O.BrokenHints)
return BrokenHints < O.BrokenHints;
}
};
- // Register mask interference. The current VirtReg is checked for register
- // mask interference on entry to selectOrSplit(). If there is no
- // interference, UsableRegs is left empty. If there is interference,
- // UsableRegs has a bit mask of registers that can be used without register
- // mask interference.
- BitVector UsableRegs;
-
- /// clobberedByRegMask - Returns true if PhysReg is not directly usable
- /// because of register mask clobbers.
- bool clobberedByRegMask(unsigned PhysReg) const {
- return !UsableRegs.empty() && !UsableRegs.test(PhysReg);
- }
-
// splitting state.
- std::auto_ptr<SplitAnalysis> SA;
- std::auto_ptr<SplitEditor> SE;
+ OwningPtr<SplitAnalysis> SA;
+ OwningPtr<SplitEditor> SE;
/// Cached per-block interference maps
InterferenceCache IntfCache;
}
};
- /// Candidate info for for each PhysReg in AllocationOrder.
+ /// Candidate info for each PhysReg in AllocationOrder.
/// This vector never shrinks, but grows to the size of the largest register
/// class.
SmallVector<GlobalSplitCandidate, 32> GlobalCand;
- enum { NoCand = ~0u };
+ enum LLVM_ENUM_INT_TYPE(unsigned) { NoCand = ~0u };
/// Candidate map. Each edge bundle is assigned to a GlobalCand entry, or to
/// NoCand which indicates the stack interval.
virtual void enqueue(LiveInterval *LI);
virtual LiveInterval *dequeue();
virtual unsigned selectOrSplit(LiveInterval&,
- SmallVectorImpl<LiveInterval*>&);
+ SmallVectorImpl<unsigned>&);
/// Perform register allocation.
virtual bool runOnMachineFunction(MachineFunction &mf);
void LRE_WillShrinkVirtReg(unsigned);
void LRE_DidCloneVirtReg(unsigned, unsigned);
- float calcSpillCost();
- bool addSplitConstraints(InterferenceCache::Cursor, float&);
+ BlockFrequency calcSpillCost();
+ bool addSplitConstraints(InterferenceCache::Cursor, BlockFrequency&);
void addThroughConstraints(InterferenceCache::Cursor, ArrayRef<unsigned>);
void growRegion(GlobalSplitCandidate &Cand);
- float calcGlobalSplitCost(GlobalSplitCandidate&);
+ BlockFrequency calcGlobalSplitCost(GlobalSplitCandidate&);
bool calcCompactRegion(GlobalSplitCandidate&);
void splitAroundRegion(LiveRangeEdit&, ArrayRef<unsigned>);
void calcGapWeights(unsigned, SmallVectorImpl<float>&);
+ unsigned canReassign(LiveInterval &VirtReg, unsigned PhysReg);
bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool);
bool canEvictInterference(LiveInterval&, unsigned, bool, EvictionCost&);
void evictInterference(LiveInterval&, unsigned,
- SmallVectorImpl<LiveInterval*>&);
+ SmallVectorImpl<unsigned>&);
unsigned tryAssign(LiveInterval&, AllocationOrder&,
- SmallVectorImpl<LiveInterval*>&);
+ SmallVectorImpl<unsigned>&);
unsigned tryEvict(LiveInterval&, AllocationOrder&,
- SmallVectorImpl<LiveInterval*>&, unsigned = ~0u);
+ SmallVectorImpl<unsigned>&, unsigned = ~0u);
unsigned tryRegionSplit(LiveInterval&, AllocationOrder&,
- SmallVectorImpl<LiveInterval*>&);
+ SmallVectorImpl<unsigned>&);
unsigned tryBlockSplit(LiveInterval&, AllocationOrder&,
- SmallVectorImpl<LiveInterval*>&);
+ SmallVectorImpl<unsigned>&);
unsigned tryInstructionSplit(LiveInterval&, AllocationOrder&,
- SmallVectorImpl<LiveInterval*>&);
+ SmallVectorImpl<unsigned>&);
unsigned tryLocalSplit(LiveInterval&, AllocationOrder&,
- SmallVectorImpl<LiveInterval*>&);
+ SmallVectorImpl<unsigned>&);
unsigned trySplit(LiveInterval&, AllocationOrder&,
- SmallVectorImpl<LiveInterval*>&);
+ SmallVectorImpl<unsigned>&);
};
} // end anonymous namespace
initializeSlotIndexesPass(*PassRegistry::getPassRegistry());
initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry());
initializeMachineSchedulerPass(*PassRegistry::getPassRegistry());
- initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry());
initializeLiveStacksPass(*PassRegistry::getPassRegistry());
initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry());
initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry());
initializeVirtRegMapPass(*PassRegistry::getPassRegistry());
+ initializeLiveRegMatrixPass(*PassRegistry::getPassRegistry());
initializeEdgeBundlesPass(*PassRegistry::getPassRegistry());
initializeSpillPlacementPass(*PassRegistry::getPassRegistry());
}
void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
+ AU.addRequired<MachineBlockFrequencyInfo>();
+ AU.addPreserved<MachineBlockFrequencyInfo>();
AU.addRequired<AliasAnalysis>();
AU.addPreserved<AliasAnalysis>();
AU.addRequired<LiveIntervals>();
+ AU.addPreserved<LiveIntervals>();
AU.addRequired<SlotIndexes>();
AU.addPreserved<SlotIndexes>();
AU.addRequired<LiveDebugVariables>();
AU.addPreserved<LiveDebugVariables>();
- AU.addRequired<CalculateSpillWeights>();
AU.addRequired<LiveStacks>();
AU.addPreserved<LiveStacks>();
AU.addRequired<MachineDominatorTree>();
AU.addPreserved<MachineLoopInfo>();
AU.addRequired<VirtRegMap>();
AU.addPreserved<VirtRegMap>();
+ AU.addRequired<LiveRegMatrix>();
+ AU.addPreserved<LiveRegMatrix>();
AU.addRequired<EdgeBundles>();
AU.addRequired<SpillPlacement>();
MachineFunctionPass::getAnalysisUsage(AU);
//===----------------------------------------------------------------------===//
bool RAGreedy::LRE_CanEraseVirtReg(unsigned VirtReg) {
- if (unsigned PhysReg = VRM->getPhys(VirtReg)) {
- unassign(LIS->getInterval(VirtReg), PhysReg);
+ if (VRM->hasPhys(VirtReg)) {
+ Matrix->unassign(LIS->getInterval(VirtReg));
return true;
}
// Unassigned virtreg is probably in the priority queue.
}
void RAGreedy::LRE_WillShrinkVirtReg(unsigned VirtReg) {
- unsigned PhysReg = VRM->getPhys(VirtReg);
- if (!PhysReg)
+ if (!VRM->hasPhys(VirtReg))
return;
// Register is assigned, put it back on the queue for reassignment.
LiveInterval &LI = LIS->getInterval(VirtReg);
- unassign(LI, PhysReg);
+ Matrix->unassign(LI);
enqueue(&LI);
}
SpillerInstance.reset(0);
ExtraRegInfo.clear();
GlobalCand.clear();
- RegAllocBase::releaseMemory();
}
void RAGreedy::enqueue(LiveInterval *LI) {
// everything else has been allocated.
Prio = Size;
} else {
- // Everything is allocated in long->short order. Long ranges that don't fit
- // should be spilled (or split) ASAP so they don't create interference.
- Prio = (1u << 31) + Size;
+ if (ExtraRegInfo[Reg].Stage == RS_Assign && !LI->empty() &&
+ LIS->intervalIsInOneMBB(*LI)) {
+ // Allocate original local ranges in linear instruction order. Since they
+ // are singly defined, this produces optimal coloring in the absence of
+ // global interference and other constraints.
+ Prio = LI->beginIndex().getInstrDistance(Indexes->getLastIndex());
+ }
+ else {
+ // Allocate global and split ranges in long->short order. Long ranges that
+ // don't fit should be spilled (or split) ASAP so they don't create
+ // interference. Mark a bit to prioritize global above local ranges.
+ Prio = (1u << 29) + Size;
+ }
+ // Mark a higher bit to prioritize global and local above RS_Split.
+ Prio |= (1u << 31);
// Boost ranges that have a physical register hint.
- if (TargetRegisterInfo::isPhysicalRegister(VRM->getRegAllocPref(Reg)))
+ if (VRM->hasKnownPreference(Reg))
Prio |= (1u << 30);
}
-
+ // The virtual register number is a tie breaker for same-sized ranges.
+ // Give lower vreg numbers higher priority to assign them first.
Queue.push(std::make_pair(Prio, ~Reg));
}
/// tryAssign - Try to assign VirtReg to an available register.
unsigned RAGreedy::tryAssign(LiveInterval &VirtReg,
AllocationOrder &Order,
- SmallVectorImpl<LiveInterval*> &NewVRegs) {
+ SmallVectorImpl<unsigned> &NewVRegs) {
Order.rewind();
unsigned PhysReg;
- while ((PhysReg = Order.next())) {
- if (clobberedByRegMask(PhysReg))
- continue;
- if (!checkPhysRegInterference(VirtReg, PhysReg))
+ while ((PhysReg = Order.next()))
+ if (!Matrix->checkInterference(VirtReg, PhysReg))
break;
- }
- if (!PhysReg || Order.isHint(PhysReg))
+ if (!PhysReg || Order.isHint())
return PhysReg;
// PhysReg is available, but there may be a better choice.
// If we missed a simple hint, try to cheaply evict interference from the
// preferred register.
if (unsigned Hint = MRI->getSimpleHint(VirtReg.reg))
- if (Order.isHint(Hint) && !clobberedByRegMask(Hint)) {
+ if (Order.isHint(Hint)) {
DEBUG(dbgs() << "missed hint " << PrintReg(Hint, TRI) << '\n');
EvictionCost MaxCost(1);
if (canEvictInterference(VirtReg, Hint, true, MaxCost)) {
// Interference eviction
//===----------------------------------------------------------------------===//
+unsigned RAGreedy::canReassign(LiveInterval &VirtReg, unsigned PrevReg) {
+ AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo);
+ unsigned PhysReg;
+ while ((PhysReg = Order.next())) {
+ if (PhysReg == PrevReg)
+ continue;
+
+ MCRegUnitIterator Units(PhysReg, TRI);
+ for (; Units.isValid(); ++Units) {
+ // Instantiate a "subquery", not to be confused with the Queries array.
+ LiveIntervalUnion::Query subQ(&VirtReg, &Matrix->getLiveUnions()[*Units]);
+ if (subQ.checkInterference())
+ break;
+ }
+ // If no units have interference, break out with the current PhysReg.
+ if (!Units.isValid())
+ break;
+ }
+ if (PhysReg)
+ DEBUG(dbgs() << "can reassign: " << VirtReg << " from "
+ << PrintReg(PrevReg, TRI) << " to " << PrintReg(PhysReg, TRI)
+ << '\n');
+ return PhysReg;
+}
+
/// shouldEvict - determine if A should evict the assigned live range B. The
/// eviction policy defined by this function together with the allocation order
/// defined by enqueue() decides which registers ultimately end up being split
///
/// @param VirtReg Live range that is about to be assigned.
/// @param PhysReg Desired register for assignment.
-/// @prarm IsHint True when PhysReg is VirtReg's preferred register.
+/// @param IsHint True when PhysReg is VirtReg's preferred register.
/// @param MaxCost Only look for cheaper candidates and update with new cost
/// when returning true.
/// @returns True when interference can be evicted cheaper than MaxCost.
bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg,
bool IsHint, EvictionCost &MaxCost) {
+ // It is only possible to evict virtual register interference.
+ if (Matrix->checkInterference(VirtReg, PhysReg) > LiveRegMatrix::IK_VirtReg)
+ return false;
+
+ bool IsLocal = LIS->intervalIsInOneMBB(VirtReg);
+
// Find VirtReg's cascade number. This will be unassigned if VirtReg was never
// involved in an eviction before. If a cascade number was assigned, deny
// evicting anything with the same or a newer cascade number. This prevents
Cascade = NextCascade;
EvictionCost Cost;
- for (MCRegAliasIterator AI(PhysReg, TRI, true); AI.isValid(); ++AI) {
- LiveIntervalUnion::Query &Q = query(VirtReg, *AI);
+ for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
+ LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units);
// If there is 10 or more interferences, chances are one is heavier.
if (Q.collectInterferingVRegs(10) >= 10)
return false;
// Check if any interfering live range is heavier than MaxWeight.
for (unsigned i = Q.interferingVRegs().size(); i; --i) {
LiveInterval *Intf = Q.interferingVRegs()[i - 1];
- if (TargetRegisterInfo::isPhysicalRegister(Intf->reg))
- return false;
+ assert(TargetRegisterInfo::isVirtualRegister(Intf->reg) &&
+ "Only expecting virtual register interference from query");
// Never evict spill products. They cannot split or spill.
if (getStage(*Intf) == RS_Done)
return false;
// Abort if this would be too expensive.
if (!(Cost < MaxCost))
return false;
+ if (Urgent)
+ continue;
+ // If !MaxCost.isMax(), then we're just looking for a cheap register.
+ // Evicting another local live range in this case could lead to suboptimal
+ // coloring.
+ if (!MaxCost.isMax() && IsLocal && LIS->intervalIsInOneMBB(*Intf) &&
+ !canReassign(*Intf, PhysReg)) {
+ return false;
+ }
// Finally, apply the eviction policy for non-urgent evictions.
- if (!Urgent && !shouldEvict(VirtReg, IsHint, *Intf, BreaksHint))
+ if (!shouldEvict(VirtReg, IsHint, *Intf, BreaksHint))
return false;
}
}
/// from being assigned to Physreg. This assumes that canEvictInterference
/// returned true.
void RAGreedy::evictInterference(LiveInterval &VirtReg, unsigned PhysReg,
- SmallVectorImpl<LiveInterval*> &NewVRegs) {
+ SmallVectorImpl<unsigned> &NewVRegs) {
// Make sure that VirtReg has a cascade number, and assign that cascade
// number to every evicted register. These live ranges than then only be
// evicted by a newer cascade, preventing infinite loops.
DEBUG(dbgs() << "evicting " << PrintReg(PhysReg, TRI)
<< " interference: Cascade " << Cascade << '\n');
- for (MCRegAliasIterator AI(PhysReg, TRI, true); AI.isValid(); ++AI) {
- LiveIntervalUnion::Query &Q = query(VirtReg, *AI);
+
+ // Collect all interfering virtregs first.
+ SmallVector<LiveInterval*, 8> Intfs;
+ for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
+ LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units);
assert(Q.seenAllInterferences() && "Didn't check all interfererences.");
- for (unsigned i = 0, e = Q.interferingVRegs().size(); i != e; ++i) {
- LiveInterval *Intf = Q.interferingVRegs()[i];
- unassign(*Intf, VRM->getPhys(Intf->reg));
- assert((ExtraRegInfo[Intf->reg].Cascade < Cascade ||
- VirtReg.isSpillable() < Intf->isSpillable()) &&
- "Cannot decrease cascade number, illegal eviction");
- ExtraRegInfo[Intf->reg].Cascade = Cascade;
- ++NumEvicted;
- NewVRegs.push_back(Intf);
- }
+ ArrayRef<LiveInterval*> IVR = Q.interferingVRegs();
+ Intfs.append(IVR.begin(), IVR.end());
+ }
+
+ // Evict them second. This will invalidate the queries.
+ for (unsigned i = 0, e = Intfs.size(); i != e; ++i) {
+ LiveInterval *Intf = Intfs[i];
+ // The same VirtReg may be present in multiple RegUnits. Skip duplicates.
+ if (!VRM->hasPhys(Intf->reg))
+ continue;
+ Matrix->unassign(*Intf);
+ assert((ExtraRegInfo[Intf->reg].Cascade < Cascade ||
+ VirtReg.isSpillable() < Intf->isSpillable()) &&
+ "Cannot decrease cascade number, illegal eviction");
+ ExtraRegInfo[Intf->reg].Cascade = Cascade;
+ ++NumEvicted;
+ NewVRegs.push_back(Intf->reg);
}
}
/// @return Physreg to assign VirtReg, or 0.
unsigned RAGreedy::tryEvict(LiveInterval &VirtReg,
AllocationOrder &Order,
- SmallVectorImpl<LiveInterval*> &NewVRegs,
+ SmallVectorImpl<unsigned> &NewVRegs,
unsigned CostPerUseLimit) {
NamedRegionTimer T("Evict", TimerGroupName, TimePassesIsEnabled);
// Keep track of the cheapest interference seen so far.
EvictionCost BestCost(~0u);
unsigned BestPhys = 0;
+ unsigned OrderLimit = Order.getOrder().size();
// When we are just looking for a reduced cost per use, don't break any
// hints, and only evict smaller spill weights.
if (CostPerUseLimit < ~0u) {
BestCost.BrokenHints = 0;
BestCost.MaxWeight = VirtReg.weight;
+
+ // Check of any registers in RC are below CostPerUseLimit.
+ const TargetRegisterClass *RC = MRI->getRegClass(VirtReg.reg);
+ unsigned MinCost = RegClassInfo.getMinCost(RC);
+ if (MinCost >= CostPerUseLimit) {
+ DEBUG(dbgs() << RC->getName() << " minimum cost = " << MinCost
+ << ", no cheaper registers to be found.\n");
+ return 0;
+ }
+
+ // It is normal for register classes to have a long tail of registers with
+ // the same cost. We don't need to look at them if they're too expensive.
+ if (TRI->getCostPerUse(Order.getOrder().back()) >= CostPerUseLimit) {
+ OrderLimit = RegClassInfo.getLastCostChange(RC);
+ DEBUG(dbgs() << "Only trying the first " << OrderLimit << " regs.\n");
+ }
}
Order.rewind();
- while (unsigned PhysReg = Order.next()) {
- if (clobberedByRegMask(PhysReg))
- continue;
+ while (unsigned PhysReg = Order.nextWithDups(OrderLimit)) {
if (TRI->getCostPerUse(PhysReg) >= CostPerUseLimit)
continue;
// The first use of a callee-saved register in a function has cost 1.
BestPhys = PhysReg;
// Stop if the hint can be used.
- if (Order.isHint(PhysReg))
+ if (Order.isHint())
break;
}
/// that all preferences in SplitConstraints are met.
/// Return false if there are no bundles with positive bias.
bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf,
- float &Cost) {
+ BlockFrequency &Cost) {
ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
// Reset interference dependent info.
SplitConstraints.resize(UseBlocks.size());
- float StaticCost = 0;
+ BlockFrequency StaticCost = 0;
for (unsigned i = 0; i != UseBlocks.size(); ++i) {
const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
SpillPlacement::BlockConstraint &BC = SplitConstraints[i];
Intf.moveToBlock(BC.Number);
BC.Entry = BI.LiveIn ? SpillPlacement::PrefReg : SpillPlacement::DontCare;
BC.Exit = BI.LiveOut ? SpillPlacement::PrefReg : SpillPlacement::DontCare;
- BC.ChangesValue = BI.FirstDef;
+ BC.ChangesValue = BI.FirstDef.isValid();
if (!Intf.hasInterference())
continue;
}
// Accumulate the total frequency of inserted spill code.
- if (Ins)
- StaticCost += Ins * SpillPlacer->getBlockFrequency(BC.Number);
+ while (Ins--)
+ StaticCost += SpillPlacer->getBlockFrequency(BC.Number);
}
Cost = StaticCost;
SpillPlacer->prepare(Cand.LiveBundles);
// The static split cost will be zero since Cand.Intf reports no interference.
- float Cost;
+ BlockFrequency Cost;
if (!addSplitConstraints(Cand.Intf, Cost)) {
DEBUG(dbgs() << ", none.\n");
return false;
/// calcSpillCost - Compute how expensive it would be to split the live range in
/// SA around all use blocks instead of forming bundle regions.
-float RAGreedy::calcSpillCost() {
- float Cost = 0;
+BlockFrequency RAGreedy::calcSpillCost() {
+ BlockFrequency Cost = 0;
ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
for (unsigned i = 0; i != UseBlocks.size(); ++i) {
const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
/// pattern in LiveBundles. This cost should be added to the local cost of the
/// interference pattern in SplitConstraints.
///
-float RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand) {
- float GlobalCost = 0;
+BlockFrequency RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand) {
+ BlockFrequency GlobalCost = 0;
const BitVector &LiveBundles = Cand.LiveBundles;
ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
for (unsigned i = 0; i != UseBlocks.size(); ++i) {
Ins += RegIn != (BC.Entry == SpillPlacement::PrefReg);
if (BI.LiveOut)
Ins += RegOut != (BC.Exit == SpillPlacement::PrefReg);
- if (Ins)
- GlobalCost += Ins * SpillPlacer->getBlockFrequency(BC.Number);
+ while (Ins--)
+ GlobalCost += SpillPlacer->getBlockFrequency(BC.Number);
}
for (unsigned i = 0, e = Cand.ActiveBlocks.size(); i != e; ++i) {
if (RegIn && RegOut) {
// We need double spill code if this block has interference.
Cand.Intf.moveToBlock(Number);
- if (Cand.Intf.hasInterference())
- GlobalCost += 2*SpillPlacer->getBlockFrequency(Number);
+ if (Cand.Intf.hasInterference()) {
+ GlobalCost += SpillPlacer->getBlockFrequency(Number);
+ GlobalCost += SpillPlacer->getBlockFrequency(Number);
+ }
continue;
}
// live-in / stack-out or stack-in live-out.
SmallVector<unsigned, 8> IntvMap;
SE->finish(&IntvMap);
- DebugVars->splitRegister(Reg, LREdit.regs());
+ DebugVars->splitRegister(Reg, LREdit.regs(), *LIS);
ExtraRegInfo.resize(MRI->getNumVirtRegs());
unsigned OrigBlocks = SA->getNumLiveBlocks();
// - Block-local splits are candidates for local splitting.
// - DCE leftovers should go back on the queue.
for (unsigned i = 0, e = LREdit.size(); i != e; ++i) {
- LiveInterval &Reg = *LREdit.get(i);
+ LiveInterval &Reg = LIS->getInterval(LREdit.get(i));
// Ignore old intervals from DCE.
if (getStage(Reg) != RS_New)
}
unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
- SmallVectorImpl<LiveInterval*> &NewVRegs) {
+ SmallVectorImpl<unsigned> &NewVRegs) {
unsigned NumCands = 0;
unsigned BestCand = NoCand;
- float BestCost;
+ BlockFrequency BestCost;
SmallVector<unsigned, 8> UsedCands;
// Check if we can split this live range around a compact region.
if (HasCompact) {
// Yes, keep GlobalCand[0] as the compact region candidate.
NumCands = 1;
- BestCost = HUGE_VALF;
+ BestCost = BlockFrequency::getMaxFrequency();
} else {
// No benefit from the compact region, our fallback will be per-block
// splitting. Make sure we find a solution that is cheaper than spilling.
- BestCost = Hysteresis * calcSpillCost();
+ BestCost = calcSpillCost();
DEBUG(dbgs() << "Cost of isolating all blocks = " << BestCost << '\n');
}
Cand.reset(IntfCache, PhysReg);
SpillPlacer->prepare(Cand.LiveBundles);
- float Cost;
+ BlockFrequency Cost;
if (!addSplitConstraints(Cand.Intf, Cost)) {
DEBUG(dbgs() << PrintReg(PhysReg, TRI) << "\tno positive bundles\n");
continue;
});
if (Cost < BestCost) {
BestCand = NumCands;
- BestCost = Hysteresis * Cost; // Prevent rounding effects.
+ BestCost = Cost;
}
++NumCands;
}
/// creates a lot of local live ranges, that will be split by tryLocalSplit if
/// they don't allocate.
unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order,
- SmallVectorImpl<LiveInterval*> &NewVRegs) {
+ SmallVectorImpl<unsigned> &NewVRegs) {
assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed");
unsigned Reg = VirtReg.reg;
bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg));
SE->finish(&IntvMap);
// Tell LiveDebugVariables about the new ranges.
- DebugVars->splitRegister(Reg, LREdit.regs());
+ DebugVars->splitRegister(Reg, LREdit.regs(), *LIS);
ExtraRegInfo.resize(MRI->getNumVirtRegs());
// Sort out the new intervals created by splitting. The remainder interval
// goes straight to spilling, the new local ranges get to stay RS_New.
for (unsigned i = 0, e = LREdit.size(); i != e; ++i) {
- LiveInterval &LI = *LREdit.get(i);
+ LiveInterval &LI = LIS->getInterval(LREdit.get(i));
if (getStage(LI) == RS_New && IntvMap[i] == 0)
setStage(LI, RS_Spill);
}
/// This is similar to spilling to a larger register class.
unsigned
RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
- SmallVectorImpl<LiveInterval*> &NewVRegs) {
+ SmallVectorImpl<unsigned> &NewVRegs) {
// There is no point to this if there are no larger sub-classes.
if (!RegClassInfo.isProperSubClass(MRI->getRegClass(VirtReg.reg)))
return 0;
SmallVector<unsigned, 8> IntvMap;
SE->finish(&IntvMap);
- DebugVars->splitRegister(VirtReg.reg, LREdit.regs());
+ DebugVars->splitRegister(VirtReg.reg, LREdit.regs(), *LIS);
ExtraRegInfo.resize(MRI->getNumVirtRegs());
// Assign all new registers to RS_Spill. This was the last chance.
GapWeight.assign(NumGaps, 0.0f);
// Add interference from each overlapping register.
- for (MCRegAliasIterator AI(PhysReg, TRI, true); AI.isValid(); ++AI) {
- if (!query(const_cast<LiveInterval&>(SA->getParent()), *AI)
- .checkInterference())
+ for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
+ if (!Matrix->query(const_cast<LiveInterval&>(SA->getParent()), *Units)
+ .checkInterference())
continue;
// We know that VirtReg is a continuous interval from FirstInstr to
// surrounding the instruction. The exception is interference before
// StartIdx and after StopIdx.
//
- LiveIntervalUnion::SegmentIter IntI = getLiveUnion(*AI).find(StartIdx);
+ LiveIntervalUnion::SegmentIter IntI =
+ Matrix->getLiveUnions()[*Units] .find(StartIdx);
for (unsigned Gap = 0; IntI.valid() && IntI.start() < StopIdx; ++IntI) {
// Skip the gaps before IntI.
while (Uses[Gap+1].getBoundaryIndex() < IntI.start())
break;
}
}
+
+ // Add fixed interference.
+ for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
+ const LiveRange &LR = LIS->getRegUnit(*Units);
+ LiveRange::const_iterator I = LR.find(StartIdx);
+ LiveRange::const_iterator E = LR.end();
+
+ // Same loop as above. Mark any overlapped gaps as HUGE_VALF.
+ for (unsigned Gap = 0; I != E && I->start < StopIdx; ++I) {
+ while (Uses[Gap+1].getBoundaryIndex() < I->start)
+ if (++Gap == NumGaps)
+ break;
+ if (Gap == NumGaps)
+ break;
+
+ for (; Gap != NumGaps; ++Gap) {
+ GapWeight[Gap] = llvm::huge_valf;
+ if (Uses[Gap+1].getBaseIndex() >= I->end)
+ break;
+ }
+ if (Gap == NumGaps)
+ break;
+ }
+ }
}
/// tryLocalSplit - Try to split VirtReg into smaller intervals inside its only
/// basic block.
///
unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
- SmallVectorImpl<LiveInterval*> &NewVRegs) {
+ SmallVectorImpl<unsigned> &NewVRegs) {
assert(SA->getUseBlocks().size() == 1 && "Not a local interval");
const SplitAnalysis::BlockInfo &BI = SA->getUseBlocks().front();
// If VirtReg is live across any register mask operands, compute a list of
// gaps with register masks.
SmallVector<unsigned, 8> RegMaskGaps;
- if (!UsableRegs.empty()) {
+ if (Matrix->checkRegMaskInterference(VirtReg)) {
// Get regmask slots for the whole block.
ArrayRef<SlotIndex> RMS = LIS->getRegMaskSlotsInBlock(BI.MBB->getNumber());
DEBUG(dbgs() << RMS.size() << " regmasks in block:");
unsigned BestAfter = 0;
float BestDiff = 0;
- const float blockFreq = SpillPlacer->getBlockFrequency(BI.MBB->getNumber());
+ const float blockFreq =
+ SpillPlacer->getBlockFrequency(BI.MBB->getNumber()).getFrequency() *
+ (1.0f / BlockFrequency::getEntryFrequency());
SmallVector<float, 8> GapWeight;
Order.rewind();
calcGapWeights(PhysReg, GapWeight);
// Remove any gaps with regmask clobbers.
- if (clobberedByRegMask(PhysReg))
+ if (Matrix->checkRegMaskInterference(VirtReg, PhysReg))
for (unsigned i = 0, e = RegMaskGaps.size(); i != e; ++i)
- GapWeight[RegMaskGaps[i]] = HUGE_VALF;
+ GapWeight[RegMaskGaps[i]] = llvm::huge_valf;
// Try to find the best sequence of gaps to close.
// The new spill weight must be larger than any gap interference.
// Legally, without causing looping?
bool Legal = !ProgressRequired || NewGaps < NumGaps;
- if (Legal && MaxGap < HUGE_VALF) {
+ if (Legal && MaxGap < llvm::huge_valf) {
// Estimate the new spill weight. Each instruction reads or writes the
// register. Conservatively assume there are no read-modify-write
// instructions.
SE->useIntv(SegStart, SegStop);
SmallVector<unsigned, 8> IntvMap;
SE->finish(&IntvMap);
- DebugVars->splitRegister(VirtReg.reg, LREdit.regs());
+ DebugVars->splitRegister(VirtReg.reg, LREdit.regs(), *LIS);
// If the new range has the same number of instructions as before, mark it as
// RS_Split2 so the next split will be forced to make progress. Otherwise,
assert(!ProgressRequired && "Didn't make progress when it was required.");
for (unsigned i = 0, e = IntvMap.size(); i != e; ++i)
if (IntvMap[i] == 1) {
- setStage(*LREdit.get(i), RS_Split2);
- DEBUG(dbgs() << PrintReg(LREdit.get(i)->reg));
+ setStage(LIS->getInterval(LREdit.get(i)), RS_Split2);
+ DEBUG(dbgs() << PrintReg(LREdit.get(i)));
}
DEBUG(dbgs() << '\n');
}
/// assignable.
/// @return Physreg when VirtReg may be assigned and/or new NewVRegs.
unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
- SmallVectorImpl<LiveInterval*>&NewVRegs) {
+ SmallVectorImpl<unsigned>&NewVRegs) {
// Ranges must be Split2 or less.
if (getStage(VirtReg) >= RS_Spill)
return 0;
// an assertion when the coalescer is fixed.
if (SA->didRepairRange()) {
// VirtReg has changed, so all cached queries are invalid.
- invalidateVirtRegs();
+ Matrix->invalidateVirtRegs();
if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs))
return PhysReg;
}
//===----------------------------------------------------------------------===//
unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
- SmallVectorImpl<LiveInterval*> &NewVRegs) {
- // Check if VirtReg is live across any calls.
- UsableRegs.clear();
- if (LIS->checkRegMaskInterference(VirtReg, UsableRegs))
- DEBUG(dbgs() << "Live across regmasks.\n");
-
+ SmallVectorImpl<unsigned> &NewVRegs) {
// First try assigning a free register.
AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo);
if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs))
if (Stage < RS_Split) {
setStage(VirtReg, RS_Split);
DEBUG(dbgs() << "wait for second round\n");
- NewVRegs.push_back(&VirtReg);
+ NewVRegs.push_back(VirtReg.reg);
return 0;
}
bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
DEBUG(dbgs() << "********** GREEDY REGISTER ALLOCATION **********\n"
- << "********** Function: "
- << ((Value*)mf.getFunction())->getName() << '\n');
+ << "********** Function: " << mf.getName() << '\n');
MF = &mf;
if (VerifyEnabled)
MF->verify(this, "Before greedy register allocator");
- RegAllocBase::init(getAnalysis<VirtRegMap>(), getAnalysis<LiveIntervals>());
+ RegAllocBase::init(getAnalysis<VirtRegMap>(),
+ getAnalysis<LiveIntervals>(),
+ getAnalysis<LiveRegMatrix>());
Indexes = &getAnalysis<SlotIndexes>();
+ MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
DomTree = &getAnalysis<MachineDominatorTree>();
SpillerInstance.reset(createInlineSpiller(*this, *MF, *VRM));
Loops = &getAnalysis<MachineLoopInfo>();
SpillPlacer = &getAnalysis<SpillPlacement>();
DebugVars = &getAnalysis<LiveDebugVariables>();
+ calculateSpillWeightsAndHints(*LIS, mf, *Loops, *MBFI);
+
+ DEBUG(LIS->dump());
+
SA.reset(new SplitAnalysis(*VRM, *LIS, *Loops));
- SE.reset(new SplitEditor(*SA, *LIS, *VRM, *DomTree));
+ SE.reset(new SplitEditor(*SA, *LIS, *VRM, *DomTree, *MBFI));
ExtraRegInfo.clear();
ExtraRegInfo.resize(MRI->getNumVirtRegs());
NextCascade = 1;
- IntfCache.init(MF, &getLiveUnion(0), Indexes, LIS, TRI);
+ IntfCache.init(MF, Matrix->getLiveUnions(), Indexes, LIS, TRI);
GlobalCand.resize(32); // This will grow as needed.
allocatePhysRegs();
- addMBBLiveIns(MF);
- LIS->addKillFlags();
-
- // Run rewriter
- {
- NamedRegionTimer T("Rewriter", TimerGroupName, TimePassesIsEnabled);
- VRM->rewrite(Indexes);
- }
-
- // Write out new DBG_VALUE instructions.
- {
- NamedRegionTimer T("Emit Debug Info", TimerGroupName, TimePassesIsEnabled);
- DebugVars->emitDebugValues(VRM);
- }
-
- // All machine operands and other references to virtual registers have been
- // replaced. Remove the virtual registers and release all the transient data.
- VRM->clearAllVirt();
- MRI->clearVirtRegs();
releaseMemory();
-
return true;
}
projects / oota-llvm.git / blobdiff