void findVRegIntervalsToAlloc(const MachineFunction &MF, LiveIntervals &LIS);
/// \brief Constructs an initial graph.
- void initializeGraph(PBQPRAGraph &G);
+ void initializeGraph(PBQPRAGraph &G, VirtRegMap &VRM, Spiller &VRegSpiller);
+
+ /// \brief Spill the given VReg.
+ void spillVReg(unsigned VReg, SmallVectorImpl<unsigned> &NewIntervals,
+ MachineFunction &MF, LiveIntervals &LIS, VirtRegMap &VRM,
+ Spiller &VRegSpiller);
/// \brief Given a solved PBQP problem maps this solution back to a register
/// assignment.
void apply(PBQPRAGraph &G) override {
LiveIntervals &LIS = G.getMetadata().LIS;
+ // A minimum spill costs, so that register constraints can can be set
+ // without normalization in the [0.0:MinSpillCost( interval.
+ const PBQP::PBQPNum MinSpillCost = 10.0;
+
for (auto NId : G.nodeIds()) {
PBQP::PBQPNum SpillCost =
LIS.getInterval(G.getNodeMetadata(NId).getVReg()).weight;
if (SpillCost == 0.0)
SpillCost = std::numeric_limits<PBQP::PBQPNum>::min();
+ else
+ SpillCost += MinSpillCost;
PBQPRAGraph::RawVector NodeCosts(G.getNodeCosts(NId));
NodeCosts[PBQP::RegAlloc::getSpillOptionIdx()] = SpillCost;
G.setNodeCosts(NId, std::move(NodeCosts));
class Interference : public PBQPRAConstraint {
private:
+ typedef const PBQP::RegAlloc::AllowedRegVector* AllowedRegVecPtr;
+ typedef std::pair<AllowedRegVecPtr, AllowedRegVecPtr> IMatrixKey;
+ typedef DenseMap<IMatrixKey, PBQPRAGraph::MatrixPtr> IMatrixCache;
+
// Holds (Interval, CurrentSegmentID, and NodeId). The first two are required
// for the fast interference graph construction algorithm. The last is there
// to save us from looking up node ids via the VRegToNode map in the graph
// number of registers, but rather the size of the largest clique in the
// graph. Still, we expect this to be better than N^2.
LiveIntervals &LIS = G.getMetadata().LIS;
- const TargetRegisterInfo &TRI =
- *G.getMetadata().MF.getTarget().getSubtargetImpl()->getRegisterInfo();
+
+ // Interferenc matrices are incredibly regular - they're only a function of
+ // the allowed sets, so we cache them to avoid the overhead of constructing
+ // and uniquing them.
+ IMatrixCache C;
typedef std::set<IntervalInfo, decltype(&lowestEndPoint)> IntervalSet;
typedef std::priority_queue<IntervalInfo, std::vector<IntervalInfo>,
// Check that we haven't already added this edge
// FIXME: findEdge is expensive in the worst case (O(max_clique(G))).
// It might be better to replace this with a local bit-matrix.
- if (G.findEdge(NId, MId) != PBQP::GraphBase::invalidEdgeId())
+ if (G.findEdge(NId, MId) != PBQPRAGraph::invalidEdgeId())
continue;
// This is a new edge - add it to the graph.
- const auto &NOpts = G.getNodeMetadata(NId).getOptionRegs();
- const auto &MOpts = G.getNodeMetadata(MId).getOptionRegs();
- G.addEdge(NId, MId, createInterferenceMatrix(TRI, NOpts, MOpts));
+ createInterferenceEdge(G, NId, MId, C);
}
// Finally, add Cur to the Active set.
private:
- PBQPRAGraph::RawMatrix createInterferenceMatrix(
- const TargetRegisterInfo &TRI,
- const PBQPRAGraph::NodeMetadata::OptionToRegMap &NOpts,
- const PBQPRAGraph::NodeMetadata::OptionToRegMap &MOpts) {
- PBQPRAGraph::RawMatrix M(NOpts.size() + 1, MOpts.size() + 1, 0);
- for (unsigned I = 0; I != NOpts.size(); ++I) {
- unsigned PRegN = NOpts[I];
- for (unsigned J = 0; J != MOpts.size(); ++J) {
- unsigned PRegM = MOpts[J];
+ void createInterferenceEdge(PBQPRAGraph &G, PBQPRAGraph::NodeId NId,
+ PBQPRAGraph::NodeId MId, IMatrixCache &C) {
+
+ const TargetRegisterInfo &TRI =
+ *G.getMetadata().MF.getSubtarget().getRegisterInfo();
+
+ const auto &NRegs = G.getNodeMetadata(NId).getAllowedRegs();
+ const auto &MRegs = G.getNodeMetadata(MId).getAllowedRegs();
+
+ // Try looking the edge costs up in the IMatrixCache first.
+ IMatrixKey K(&NRegs, &MRegs);
+ IMatrixCache::iterator I = C.find(K);
+ if (I != C.end()) {
+ G.addEdgeBypassingCostAllocator(NId, MId, I->second);
+ return;
+ }
+
+ PBQPRAGraph::RawMatrix M(NRegs.size() + 1, MRegs.size() + 1, 0);
+ for (unsigned I = 0; I != NRegs.size(); ++I) {
+ unsigned PRegN = NRegs[I];
+ for (unsigned J = 0; J != MRegs.size(); ++J) {
+ unsigned PRegM = MRegs[J];
if (TRI.regsOverlap(PRegN, PRegM))
M[I + 1][J + 1] = std::numeric_limits<PBQP::PBQPNum>::infinity();
}
}
- return M;
+ PBQPRAGraph::EdgeId EId = G.addEdge(NId, MId, std::move(M));
+ C[K] = G.getEdgeCostsPtr(EId);
}
};
void apply(PBQPRAGraph &G) override {
MachineFunction &MF = G.getMetadata().MF;
MachineBlockFrequencyInfo &MBFI = G.getMetadata().MBFI;
- CoalescerPair CP(*MF.getTarget().getSubtargetImpl()->getRegisterInfo());
+ CoalescerPair CP(*MF.getSubtarget().getRegisterInfo());
// Scan the machine function and add a coalescing cost whenever CoalescerPair
// gives the Ok.
unsigned DstReg = CP.getDstReg();
unsigned SrcReg = CP.getSrcReg();
- const float CopyFactor = 0.5; // Cost of copy relative to load. Current
- // value plucked randomly out of the air.
-
- PBQP::PBQPNum CBenefit =
- CopyFactor * LiveIntervals::getSpillWeight(false, true, &MBFI, &MI);
+ const float Scale = 1.0f / MBFI.getEntryFreq();
+ PBQP::PBQPNum CBenefit = MBFI.getBlockFreq(&MBB).getFrequency() * Scale;
if (CP.isPhys()) {
if (!MF.getRegInfo().isAllocatable(DstReg))
PBQPRAGraph::NodeId NId = G.getMetadata().getNodeIdForVReg(SrcReg);
- const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed =
- G.getNodeMetadata(NId).getOptionRegs();
+ const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed =
+ G.getNodeMetadata(NId).getAllowedRegs();
unsigned PRegOpt = 0;
while (PRegOpt < Allowed.size() && Allowed[PRegOpt] != DstReg)
if (PRegOpt < Allowed.size()) {
PBQPRAGraph::RawVector NewCosts(G.getNodeCosts(NId));
- NewCosts[PRegOpt + 1] += CBenefit;
+ NewCosts[PRegOpt + 1] -= CBenefit;
G.setNodeCosts(NId, std::move(NewCosts));
}
} else {
PBQPRAGraph::NodeId N1Id = G.getMetadata().getNodeIdForVReg(DstReg);
PBQPRAGraph::NodeId N2Id = G.getMetadata().getNodeIdForVReg(SrcReg);
- const PBQPRAGraph::NodeMetadata::OptionToRegMap *Allowed1 =
- &G.getNodeMetadata(N1Id).getOptionRegs();
- const PBQPRAGraph::NodeMetadata::OptionToRegMap *Allowed2 =
- &G.getNodeMetadata(N2Id).getOptionRegs();
+ const PBQPRAGraph::NodeMetadata::AllowedRegVector *Allowed1 =
+ &G.getNodeMetadata(N1Id).getAllowedRegs();
+ const PBQPRAGraph::NodeMetadata::AllowedRegVector *Allowed2 =
+ &G.getNodeMetadata(N2Id).getAllowedRegs();
PBQPRAGraph::EdgeId EId = G.findEdge(N1Id, N2Id);
if (EId == G.invalidEdgeId()) {
private:
void addVirtRegCoalesce(
- PBQPRAGraph::RawMatrix &CostMat,
- const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed1,
- const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed2,
- PBQP::PBQPNum Benefit) {
+ PBQPRAGraph::RawMatrix &CostMat,
+ const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed1,
+ const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed2,
+ PBQP::PBQPNum Benefit) {
assert(CostMat.getRows() == Allowed1.size() + 1 && "Size mismatch.");
assert(CostMat.getCols() == Allowed2.size() + 1 && "Size mismatch.");
for (unsigned I = 0; I != Allowed1.size(); ++I) {
for (unsigned J = 0; J != Allowed2.size(); ++J) {
unsigned PReg2 = Allowed2[J];
if (PReg1 == PReg2)
- CostMat[I + 1][J + 1] += -Benefit;
+ CostMat[I + 1][J + 1] -= Benefit;
}
}
}
}
}
-void RegAllocPBQP::initializeGraph(PBQPRAGraph &G) {
+static bool isACalleeSavedRegister(unsigned reg, const TargetRegisterInfo &TRI,
+ const MachineFunction &MF) {
+ const MCPhysReg *CSR = TRI.getCalleeSavedRegs(&MF);
+ for (unsigned i = 0; CSR[i] != 0; ++i)
+ if (TRI.regsOverlap(reg, CSR[i]))
+ return true;
+ return false;
+}
+
+void RegAllocPBQP::initializeGraph(PBQPRAGraph &G, VirtRegMap &VRM,
+ Spiller &VRegSpiller) {
MachineFunction &MF = G.getMetadata().MF;
LiveIntervals &LIS = G.getMetadata().LIS;
const MachineRegisterInfo &MRI = G.getMetadata().MF.getRegInfo();
const TargetRegisterInfo &TRI =
- *G.getMetadata().MF.getTarget().getSubtargetImpl()->getRegisterInfo();
+ *G.getMetadata().MF.getSubtarget().getRegisterInfo();
+
+ std::vector<unsigned> Worklist(VRegsToAlloc.begin(), VRegsToAlloc.end());
+
+ while (!Worklist.empty()) {
+ unsigned VReg = Worklist.back();
+ Worklist.pop_back();
- for (auto VReg : VRegsToAlloc) {
const TargetRegisterClass *TRC = MRI.getRegClass(VReg);
LiveInterval &VRegLI = LIS.getInterval(VReg);
VRegAllowed.push_back(PReg);
}
+ // Check for vregs that have no allowed registers. These should be
+ // pre-spilled and the new vregs added to the worklist.
+ if (VRegAllowed.empty()) {
+ SmallVector<unsigned, 8> NewVRegs;
+ spillVReg(VReg, NewVRegs, MF, LIS, VRM, VRegSpiller);
+ for (auto NewVReg : NewVRegs)
+ Worklist.push_back(NewVReg);
+ continue;
+ }
+
PBQPRAGraph::RawVector NodeCosts(VRegAllowed.size() + 1, 0);
+
+ // Tweak cost of callee saved registers, as using then force spilling and
+ // restoring them. This would only happen in the prologue / epilogue though.
+ for (unsigned i = 0; i != VRegAllowed.size(); ++i)
+ if (isACalleeSavedRegister(VRegAllowed[i], TRI, MF))
+ NodeCosts[1 + i] += 1.0;
+
PBQPRAGraph::NodeId NId = G.addNode(std::move(NodeCosts));
G.getNodeMetadata(NId).setVReg(VReg);
- G.getNodeMetadata(NId).setOptionRegs(std::move(VRegAllowed));
+ G.getNodeMetadata(NId).setAllowedRegs(
+ G.getMetadata().getAllowedRegs(std::move(VRegAllowed)));
G.getMetadata().setNodeIdForVReg(VReg, NId);
}
}
+void RegAllocPBQP::spillVReg(unsigned VReg,
+ SmallVectorImpl<unsigned> &NewIntervals,
+ MachineFunction &MF, LiveIntervals &LIS,
+ VirtRegMap &VRM, Spiller &VRegSpiller) {
+
+ VRegsToAlloc.erase(VReg);
+ LiveRangeEdit LRE(&LIS.getInterval(VReg), NewIntervals, MF, LIS, &VRM);
+ VRegSpiller.spill(LRE);
+
+ const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
+ (void)TRI;
+ DEBUG(dbgs() << "VREG " << PrintReg(VReg, &TRI) << " -> SPILLED (Cost: "
+ << LRE.getParent().weight << ", New vregs: ");
+
+ // Copy any newly inserted live intervals into the list of regs to
+ // allocate.
+ for (LiveRangeEdit::iterator I = LRE.begin(), E = LRE.end();
+ I != E; ++I) {
+ const LiveInterval &LI = LIS.getInterval(*I);
+ assert(!LI.empty() && "Empty spill range.");
+ DEBUG(dbgs() << PrintReg(LI.reg, &TRI) << " ");
+ VRegsToAlloc.insert(LI.reg);
+ }
+
+ DEBUG(dbgs() << ")\n");
+}
+
bool RegAllocPBQP::mapPBQPToRegAlloc(const PBQPRAGraph &G,
const PBQP::Solution &Solution,
VirtRegMap &VRM,
Spiller &VRegSpiller) {
MachineFunction &MF = G.getMetadata().MF;
LiveIntervals &LIS = G.getMetadata().LIS;
- const TargetRegisterInfo &TRI =
- *MF.getTarget().getSubtargetImpl()->getRegisterInfo();
+ const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
(void)TRI;
// Set to true if we have any spills
unsigned AllocOption = Solution.getSelection(NId);
if (AllocOption != PBQP::RegAlloc::getSpillOptionIdx()) {
- unsigned PReg = G.getNodeMetadata(NId).getOptionRegs()[AllocOption - 1];
+ unsigned PReg = G.getNodeMetadata(NId).getAllowedRegs()[AllocOption - 1];
DEBUG(dbgs() << "VREG " << PrintReg(VReg, &TRI) << " -> "
<< TRI.getName(PReg) << "\n");
assert(PReg != 0 && "Invalid preg selected.");
VRM.assignVirt2Phys(VReg, PReg);
} else {
- VRegsToAlloc.erase(VReg);
- SmallVector<unsigned, 8> NewSpills;
- LiveRangeEdit LRE(&LIS.getInterval(VReg), NewSpills, MF, LIS, &VRM);
- VRegSpiller.spill(LRE);
-
- DEBUG(dbgs() << "VREG " << PrintReg(VReg, &TRI) << " -> SPILLED (Cost: "
- << LRE.getParent().weight << ", New vregs: ");
-
- // Copy any newly inserted live intervals into the list of regs to
- // allocate.
- for (LiveRangeEdit::iterator I = LRE.begin(), E = LRE.end();
- I != E; ++I) {
- LiveInterval &LI = LIS.getInterval(*I);
- assert(!LI.empty() && "Empty spill range.");
- DEBUG(dbgs() << PrintReg(LI.reg, &TRI) << " ");
- VRegsToAlloc.insert(LI.reg);
- }
-
- DEBUG(dbgs() << ")\n");
-
- // We need another round if spill intervals were added.
- AnotherRoundNeeded |= !LRE.empty();
+ // Spill VReg. If this introduces new intervals we'll need another round
+ // of allocation.
+ SmallVector<unsigned, 8> NewVRegs;
+ spillVReg(VReg, NewVRegs, MF, LIS, VRM, VRegSpiller);
+ AnotherRoundNeeded |= !NewVRegs.empty();
}
}
return !AnotherRoundNeeded;
}
-
void RegAllocPBQP::finalizeAlloc(MachineFunction &MF,
LiveIntervals &LIS,
VirtRegMap &VRM) const {
}
}
+static inline float normalizePBQPSpillWeight(float UseDefFreq, unsigned Size,
+ unsigned NumInstr) {
+ // All intervals have a spill weight that is mostly proportional to the number
+ // of uses, with uses in loops having a bigger weight.
+ return NumInstr * normalizeSpillWeight(UseDefFreq, Size, 1);
+}
+
bool RegAllocPBQP::runOnMachineFunction(MachineFunction &MF) {
LiveIntervals &LIS = getAnalysis<LiveIntervals>();
MachineBlockFrequencyInfo &MBFI =
getAnalysis<MachineBlockFrequencyInfo>();
- calculateSpillWeightsAndHints(LIS, MF, getAnalysis<MachineLoopInfo>(), MBFI);
+ calculateSpillWeightsAndHints(LIS, MF, getAnalysis<MachineLoopInfo>(), MBFI,
+ normalizePBQPSpillWeight);
VirtRegMap &VRM = getAnalysis<VirtRegMap>();
// If there are non-empty intervals allocate them using pbqp.
if (!VRegsToAlloc.empty()) {
- const TargetSubtargetInfo &Subtarget = *MF.getTarget().getSubtargetImpl();
+ const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
std::unique_ptr<PBQPRAConstraintList> ConstraintsRoot =
llvm::make_unique<PBQPRAConstraintList>();
ConstraintsRoot->addConstraint(llvm::make_unique<SpillCosts>());
DEBUG(dbgs() << " PBQP Regalloc round " << Round << ":\n");
PBQPRAGraph G(PBQPRAGraph::GraphMetadata(MF, LIS, MBFI));
- initializeGraph(G);
+ initializeGraph(G, VRM, *VRegSpiller);
ConstraintsRoot->apply(G);
#ifndef NDEBUG
raw_fd_ostream OS(GraphFileName, EC, sys::fs::F_Text);
DEBUG(dbgs() << "Dumping graph for round " << Round << " to \""
<< GraphFileName << "\"\n");
- G.dumpToStream(OS);
+ G.dump(OS);
}
#endif
return true;
}
+namespace {
+// A helper class for printing node and register info in a consistent way
+class PrintNodeInfo {
+public:
+ typedef PBQP::RegAlloc::PBQPRAGraph Graph;
+ typedef PBQP::RegAlloc::PBQPRAGraph::NodeId NodeId;
+
+ PrintNodeInfo(NodeId NId, const Graph &G) : G(G), NId(NId) {}
+
+ void print(raw_ostream &OS) const {
+ const MachineRegisterInfo &MRI = G.getMetadata().MF.getRegInfo();
+ const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
+ unsigned VReg = G.getNodeMetadata(NId).getVReg();
+ const char *RegClassName = TRI->getRegClassName(MRI.getRegClass(VReg));
+ OS << NId << " (" << RegClassName << ':' << PrintReg(VReg, TRI) << ')';
+ }
+
+private:
+ const Graph &G;
+ NodeId NId;
+};
+
+inline raw_ostream &operator<<(raw_ostream &OS, const PrintNodeInfo &PR) {
+ PR.print(OS);
+ return OS;
+}
+} // anonymous namespace
+
+void PBQP::RegAlloc::PBQPRAGraph::dump(raw_ostream &OS) const {
+ for (auto NId : nodeIds()) {
+ const Vector &Costs = getNodeCosts(NId);
+ assert(Costs.getLength() != 0 && "Empty vector in graph.");
+ OS << PrintNodeInfo(NId, *this) << ": " << Costs << '\n';
+ }
+ OS << '\n';
+
+ for (auto EId : edgeIds()) {
+ NodeId N1Id = getEdgeNode1Id(EId);
+ NodeId N2Id = getEdgeNode2Id(EId);
+ assert(N1Id != N2Id && "PBQP graphs should not have self-edges.");
+ const Matrix &M = getEdgeCosts(EId);
+ assert(M.getRows() != 0 && "No rows in matrix.");
+ assert(M.getCols() != 0 && "No cols in matrix.");
+ OS << PrintNodeInfo(N1Id, *this) << ' ' << M.getRows() << " rows / ";
+ OS << PrintNodeInfo(N2Id, *this) << ' ' << M.getCols() << " cols:\n";
+ OS << M << '\n';
+ }
+}
+
+void PBQP::RegAlloc::PBQPRAGraph::dump() const { dump(dbgs()); }
+
+void PBQP::RegAlloc::PBQPRAGraph::printDot(raw_ostream &OS) const {
+ OS << "graph {\n";
+ for (auto NId : nodeIds()) {
+ OS << " node" << NId << " [ label=\""
+ << PrintNodeInfo(NId, *this) << "\\n"
+ << getNodeCosts(NId) << "\" ]\n";
+ }
+
+ OS << " edge [ len=" << nodeIds().size() << " ]\n";
+ for (auto EId : edgeIds()) {
+ OS << " node" << getEdgeNode1Id(EId)
+ << " -- node" << getEdgeNode2Id(EId)
+ << " [ label=\"";
+ const Matrix &EdgeCosts = getEdgeCosts(EId);
+ for (unsigned i = 0; i < EdgeCosts.getRows(); ++i) {
+ OS << EdgeCosts.getRowAsVector(i) << "\\n";
+ }
+ OS << "\" ]\n";
+ }
+ OS << "}\n";
+}
+
FunctionPass *llvm::createPBQPRegisterAllocator(char *customPassID) {
return new RegAllocPBQP(customPassID);
}