+ }
+
+ // FIXME: Re-implementing default behavior to work around MSVC. Remove once
+ // MSVC synthesizes move constructors properly.
+ NodeMetadata(NodeMetadata &&Other)
+ : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts),
+ OptUnsafeEdges(std::move(Other.OptUnsafeEdges)), VReg(Other.VReg),
+ AllowedRegs(std::move(Other.AllowedRegs))
+#ifndef NDEBUG
+ , everConservativelyAllocatable(Other.everConservativelyAllocatable)
+#endif
+ {}
+
+ // FIXME: Re-implementing default behavior to work around MSVC. Remove once
+ // MSVC synthesizes move constructors properly.
+ NodeMetadata& operator=(const NodeMetadata &Other) {
+ RS = Other.RS;
+ NumOpts = Other.NumOpts;
+ DeniedOpts = Other.DeniedOpts;
+ OptUnsafeEdges.reset(new unsigned[NumOpts]);
+ std::copy(Other.OptUnsafeEdges.get(), Other.OptUnsafeEdges.get() + NumOpts,
+ OptUnsafeEdges.get());
+ VReg = Other.VReg;
+ AllowedRegs = Other.AllowedRegs;
+#ifndef NDEBUG
+ everConservativelyAllocatable = Other.everConservativelyAllocatable;
+#endif
+ return *this;
+ }
+
+ // FIXME: Re-implementing default behavior to work around MSVC. Remove once
+ // MSVC synthesizes move constructors properly.
+ NodeMetadata& operator=(NodeMetadata &&Other) {
+ RS = Other.RS;
+ NumOpts = Other.NumOpts;
+ DeniedOpts = Other.DeniedOpts;
+ OptUnsafeEdges = std::move(Other.OptUnsafeEdges);
+ VReg = Other.VReg;
+ AllowedRegs = std::move(Other.AllowedRegs);
+#ifndef NDEBUG
+ everConservativelyAllocatable = Other.everConservativelyAllocatable;
+#endif
+ return *this;
+ }
+
+ void setVReg(unsigned VReg) { this->VReg = VReg; }
+ unsigned getVReg() const { return VReg; }
+
+ void setAllowedRegs(GraphMetadata::AllowedRegVecRef AllowedRegs) {
+ this->AllowedRegs = std::move(AllowedRegs);
+ }
+ const AllowedRegVector& getAllowedRegs() const { return *AllowedRegs; }
+
+ void setup(const Vector& Costs) {
+ NumOpts = Costs.getLength() - 1;
+ OptUnsafeEdges = std::unique_ptr<unsigned[]>(new unsigned[NumOpts]());
+ }
+
+ ReductionState getReductionState() const { return RS; }
+ void setReductionState(ReductionState RS) {
+ assert(RS >= this->RS && "A node's reduction state can not be downgraded");
+ this->RS = RS;
+
+#ifndef NDEBUG
+ // Remember this state to assert later that a non-infinite register
+ // option was available.
+ if (RS == ConservativelyAllocatable)
+ everConservativelyAllocatable = true;
+#endif
+ }
+
+
+ void handleAddEdge(const MatrixMetadata& MD, bool Transpose) {
+ DeniedOpts += Transpose ? MD.getWorstRow() : MD.getWorstCol();
+ const bool* UnsafeOpts =
+ Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows();
+ for (unsigned i = 0; i < NumOpts; ++i)
+ OptUnsafeEdges[i] += UnsafeOpts[i];
+ }
+
+ void handleRemoveEdge(const MatrixMetadata& MD, bool Transpose) {
+ DeniedOpts -= Transpose ? MD.getWorstRow() : MD.getWorstCol();
+ const bool* UnsafeOpts =
+ Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows();
+ for (unsigned i = 0; i < NumOpts; ++i)
+ OptUnsafeEdges[i] -= UnsafeOpts[i];
+ }
+
+ bool isConservativelyAllocatable() const {
+ return (DeniedOpts < NumOpts) ||
+ (std::find(&OptUnsafeEdges[0], &OptUnsafeEdges[NumOpts], 0) !=
+ &OptUnsafeEdges[NumOpts]);
+ }
+
+#ifndef NDEBUG
+ bool wasConservativelyAllocatable() const {
+ return everConservativelyAllocatable;
+ }
+#endif
+
+private:
+ ReductionState RS;
+ unsigned NumOpts;
+ unsigned DeniedOpts;
+ std::unique_ptr<unsigned[]> OptUnsafeEdges;
+ unsigned VReg;
+ GraphMetadata::AllowedRegVecRef AllowedRegs;
+
+#ifndef NDEBUG
+ bool everConservativelyAllocatable;
+#endif
+};
+
+class RegAllocSolverImpl {
+private:
+ typedef MDMatrix<MatrixMetadata> RAMatrix;
+public:
+ typedef PBQP::Vector RawVector;
+ typedef PBQP::Matrix RawMatrix;
+ typedef PBQP::Vector Vector;
+ typedef RAMatrix Matrix;
+ typedef PBQP::PoolCostAllocator<Vector, Matrix> CostAllocator;
+
+ typedef GraphBase::NodeId NodeId;
+ typedef GraphBase::EdgeId EdgeId;
+
+ typedef RegAlloc::NodeMetadata NodeMetadata;
+ struct EdgeMetadata { };
+ typedef RegAlloc::GraphMetadata GraphMetadata;
+
+ typedef PBQP::Graph<RegAllocSolverImpl> Graph;
+
+ RegAllocSolverImpl(Graph &G) : G(G) {}
+
+ Solution solve() {
+ G.setSolver(*this);
+ Solution S;
+ setup();
+ S = backpropagate(G, reduce());
+ G.unsetSolver();
+ return S;
+ }
+
+ void handleAddNode(NodeId NId) {
+ assert(G.getNodeCosts(NId).getLength() > 1 &&
+ "PBQP Graph should not contain single or zero-option nodes");
+ G.getNodeMetadata(NId).setup(G.getNodeCosts(NId));
+ }
+ void handleRemoveNode(NodeId NId) {}
+ void handleSetNodeCosts(NodeId NId, const Vector& newCosts) {}
+
+ void handleAddEdge(EdgeId EId) {
+ handleReconnectEdge(EId, G.getEdgeNode1Id(EId));
+ handleReconnectEdge(EId, G.getEdgeNode2Id(EId));
+ }
+
+ void handleRemoveEdge(EdgeId EId) {
+ handleDisconnectEdge(EId, G.getEdgeNode1Id(EId));
+ handleDisconnectEdge(EId, G.getEdgeNode2Id(EId));
+ }
+
+ void handleDisconnectEdge(EdgeId EId, NodeId NId) {
+ NodeMetadata& NMd = G.getNodeMetadata(NId);
+ const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata();
+ NMd.handleRemoveEdge(MMd, NId == G.getEdgeNode2Id(EId));
+ promote(NId, NMd);
+ }
+
+ void handleReconnectEdge(EdgeId EId, NodeId NId) {
+ NodeMetadata& NMd = G.getNodeMetadata(NId);
+ const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata();
+ NMd.handleAddEdge(MMd, NId == G.getEdgeNode2Id(EId));
+ }
+
+ void handleUpdateCosts(EdgeId EId, const Matrix& NewCosts) {
+ NodeId N1Id = G.getEdgeNode1Id(EId);
+ NodeId N2Id = G.getEdgeNode2Id(EId);
+ NodeMetadata& N1Md = G.getNodeMetadata(N1Id);
+ NodeMetadata& N2Md = G.getNodeMetadata(N2Id);
+ bool Transpose = N1Id != G.getEdgeNode1Id(EId);
+
+ // Metadata are computed incrementally. First, update them
+ // by removing the old cost.
+ const MatrixMetadata& OldMMd = G.getEdgeCosts(EId).getMetadata();
+ N1Md.handleRemoveEdge(OldMMd, Transpose);
+ N2Md.handleRemoveEdge(OldMMd, !Transpose);
+
+ // And update now the metadata with the new cost.
+ const MatrixMetadata& MMd = NewCosts.getMetadata();
+ N1Md.handleAddEdge(MMd, Transpose);
+ N2Md.handleAddEdge(MMd, !Transpose);
+
+ // As the metadata may have changed with the update, the nodes may have
+ // become ConservativelyAllocatable or OptimallyReducible.
+ promote(N1Id, N1Md);
+ promote(N2Id, N2Md);
+ }
+
+private:
+
+ void promote(NodeId NId, NodeMetadata& NMd) {
+ if (G.getNodeDegree(NId) == 3) {
+ // This node is becoming optimally reducible.
+ moveToOptimallyReducibleNodes(NId);
+ } else if (NMd.getReductionState() ==
+ NodeMetadata::NotProvablyAllocatable &&
+ NMd.isConservativelyAllocatable()) {
+ // This node just became conservatively allocatable.
+ moveToConservativelyAllocatableNodes(NId);
+ }
+ }
+
+ void removeFromCurrentSet(NodeId NId) {
+ switch (G.getNodeMetadata(NId).getReductionState()) {
+ case NodeMetadata::Unprocessed: break;
+ case NodeMetadata::OptimallyReducible:
+ assert(OptimallyReducibleNodes.find(NId) !=
+ OptimallyReducibleNodes.end() &&
+ "Node not in optimally reducible set.");
+ OptimallyReducibleNodes.erase(NId);
+ break;
+ case NodeMetadata::ConservativelyAllocatable:
+ assert(ConservativelyAllocatableNodes.find(NId) !=
+ ConservativelyAllocatableNodes.end() &&
+ "Node not in conservatively allocatable set.");
+ ConservativelyAllocatableNodes.erase(NId);
+ break;
+ case NodeMetadata::NotProvablyAllocatable:
+ assert(NotProvablyAllocatableNodes.find(NId) !=
+ NotProvablyAllocatableNodes.end() &&
+ "Node not in not-provably-allocatable set.");
+ NotProvablyAllocatableNodes.erase(NId);
+ break;
+ }
+ }
+
+ void moveToOptimallyReducibleNodes(NodeId NId) {
+ removeFromCurrentSet(NId);
+ OptimallyReducibleNodes.insert(NId);
+ G.getNodeMetadata(NId).setReductionState(
+ NodeMetadata::OptimallyReducible);
+ }
+
+ void moveToConservativelyAllocatableNodes(NodeId NId) {
+ removeFromCurrentSet(NId);
+ ConservativelyAllocatableNodes.insert(NId);
+ G.getNodeMetadata(NId).setReductionState(
+ NodeMetadata::ConservativelyAllocatable);
+ }
+
+ void moveToNotProvablyAllocatableNodes(NodeId NId) {
+ removeFromCurrentSet(NId);
+ NotProvablyAllocatableNodes.insert(NId);
+ G.getNodeMetadata(NId).setReductionState(
+ NodeMetadata::NotProvablyAllocatable);
+ }
+
+ void setup() {
+ // Set up worklists.
+ for (auto NId : G.nodeIds()) {
+ if (G.getNodeDegree(NId) < 3)
+ moveToOptimallyReducibleNodes(NId);
+ else if (G.getNodeMetadata(NId).isConservativelyAllocatable())
+ moveToConservativelyAllocatableNodes(NId);
+ else
+ moveToNotProvablyAllocatableNodes(NId);
+ }
+ }
+
+ // Compute a reduction order for the graph by iteratively applying PBQP
+ // reduction rules. Locally optimal rules are applied whenever possible (R0,
+ // R1, R2). If no locally-optimal rules apply then any conservatively
+ // allocatable node is reduced. Finally, if no conservatively allocatable
+ // node exists then the node with the lowest spill-cost:degree ratio is
+ // selected.
+ std::vector<GraphBase::NodeId> reduce() {
+ assert(!G.empty() && "Cannot reduce empty graph.");
+
+ typedef GraphBase::NodeId NodeId;
+ std::vector<NodeId> NodeStack;
+
+ // Consume worklists.
+ while (true) {
+ if (!OptimallyReducibleNodes.empty()) {
+ NodeSet::iterator NItr = OptimallyReducibleNodes.begin();
+ NodeId NId = *NItr;
+ OptimallyReducibleNodes.erase(NItr);
+ NodeStack.push_back(NId);
+ switch (G.getNodeDegree(NId)) {
+ case 0:
+ break;
+ case 1:
+ applyR1(G, NId);
+ break;
+ case 2:
+ applyR2(G, NId);
+ break;
+ default: llvm_unreachable("Not an optimally reducible node.");
+ }
+ } else if (!ConservativelyAllocatableNodes.empty()) {
+ // Conservatively allocatable nodes will never spill. For now just
+ // take the first node in the set and push it on the stack. When we
+ // start optimizing more heavily for register preferencing, it may
+ // would be better to push nodes with lower 'expected' or worst-case
+ // register costs first (since early nodes are the most
+ // constrained).
+ NodeSet::iterator NItr = ConservativelyAllocatableNodes.begin();
+ NodeId NId = *NItr;
+ ConservativelyAllocatableNodes.erase(NItr);
+ NodeStack.push_back(NId);
+ G.disconnectAllNeighborsFromNode(NId);
+
+ } else if (!NotProvablyAllocatableNodes.empty()) {
+ NodeSet::iterator NItr =
+ std::min_element(NotProvablyAllocatableNodes.begin(),
+ NotProvablyAllocatableNodes.end(),
+ SpillCostComparator(G));
+ NodeId NId = *NItr;
+ NotProvablyAllocatableNodes.erase(NItr);
+ NodeStack.push_back(NId);
+ G.disconnectAllNeighborsFromNode(NId);
+ } else
+ break;