From: Dan Gohman Date: Wed, 19 Nov 2008 23:18:57 +0000 (+0000) Subject: Experimental post-pass scheduling support. Post-pass scheduling X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=343f0c046702831a4a6aec951b6a297a23241a55;p=oota-llvm.git Experimental post-pass scheduling support. Post-pass scheduling is currently off by default, and can be enabled with -disable-post-RA-scheduler=false. This doesn't have a significant impact on most code yet because it doesn't yet do anything to address anti-dependencies and it doesn't attempt to disambiguate memory references. Also, several popular targets don't have pipeline descriptions yet. The majority of the changes here are splitting the SelectionDAG-specific code out of ScheduleDAG, so that ScheduleDAG can be moved to libLLVMCodeGen.a. The interface between ScheduleDAG-using code and the rest of the scheduling code is somewhat rough and will evolve. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@59676 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/llvm/CodeGen/LatencyPriorityQueue.h b/include/llvm/CodeGen/LatencyPriorityQueue.h new file mode 100644 index 00000000000..f04d2ede5ac --- /dev/null +++ b/include/llvm/CodeGen/LatencyPriorityQueue.h @@ -0,0 +1,124 @@ +//===---- LatencyPriorityQueue.h - A latency-oriented priority queue ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares the LatencyPriorityQueue class, which is a +// SchedulingPriorityQueue that schedules using latency information to +// reduce the length of the critical path through the basic block. +// +//===----------------------------------------------------------------------===// + +#ifndef LATENCY_PRIORITY_QUEUE_H +#define LATENCY_PRIORITY_QUEUE_H + +#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/ADT/PriorityQueue.h" + +namespace llvm { + class LatencyPriorityQueue; + + /// Sorting functions for the Available queue. + struct latency_sort : public std::binary_function { + LatencyPriorityQueue *PQ; + explicit latency_sort(LatencyPriorityQueue *pq) : PQ(pq) {} + + bool operator()(const SUnit* left, const SUnit* right) const; + }; + + class LatencyPriorityQueue : public SchedulingPriorityQueue { + // SUnits - The SUnits for the current graph. + std::vector *SUnits; + + // Latencies - The latency (max of latency from this node to the bb exit) + // for each node. + std::vector Latencies; + + /// NumNodesSolelyBlocking - This vector contains, for every node in the + /// Queue, the number of nodes that the node is the sole unscheduled + /// predecessor for. This is used as a tie-breaker heuristic for better + /// mobility. + std::vector NumNodesSolelyBlocking; + + PriorityQueue, latency_sort> Queue; +public: + LatencyPriorityQueue() : Queue(latency_sort(this)) { + } + + void initNodes(std::vector &sunits) { + SUnits = &sunits; + // Calculate node priorities. + CalculatePriorities(); + } + + void addNode(const SUnit *SU) { + Latencies.resize(SUnits->size(), -1); + NumNodesSolelyBlocking.resize(SUnits->size(), 0); + CalcLatency(*SU); + } + + void updateNode(const SUnit *SU) { + Latencies[SU->NodeNum] = -1; + CalcLatency(*SU); + } + + void releaseState() { + SUnits = 0; + Latencies.clear(); + } + + unsigned getLatency(unsigned NodeNum) const { + assert(NodeNum < Latencies.size()); + return Latencies[NodeNum]; + } + + unsigned getNumSolelyBlockNodes(unsigned NodeNum) const { + assert(NodeNum < NumNodesSolelyBlocking.size()); + return NumNodesSolelyBlocking[NodeNum]; + } + + unsigned size() const { return Queue.size(); } + + bool empty() const { return Queue.empty(); } + + virtual void push(SUnit *U) { + push_impl(U); + } + void push_impl(SUnit *U); + + void push_all(const std::vector &Nodes) { + for (unsigned i = 0, e = Nodes.size(); i != e; ++i) + push_impl(Nodes[i]); + } + + SUnit *pop() { + if (empty()) return NULL; + SUnit *V = Queue.top(); + Queue.pop(); + return V; + } + + void remove(SUnit *SU) { + assert(!Queue.empty() && "Not in queue!"); + Queue.erase_one(SU); + } + + // ScheduledNode - As nodes are scheduled, we look to see if there are any + // successor nodes that have a single unscheduled predecessor. If so, that + // single predecessor has a higher priority, since scheduling it will make + // the node available. + void ScheduledNode(SUnit *Node); + +private: + void CalculatePriorities(); + int CalcLatency(const SUnit &SU); + void AdjustPriorityOfUnscheduledPreds(SUnit *SU); + SUnit *getSingleUnscheduledPred(SUnit *SU); + }; +} + +#endif diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h index 285350057d5..610d66b3772 100644 --- a/include/llvm/CodeGen/LinkAllCodegenComponents.h +++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h @@ -16,7 +16,7 @@ #define LLVM_CODEGEN_LINKALLCODEGENCOMPONENTS_H #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" #include "llvm/CodeGen/GCs.h" namespace { diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h index 49dabe67580..ee252033ada 100644 --- a/include/llvm/CodeGen/ScheduleDAG.h +++ b/include/llvm/CodeGen/ScheduleDAG.h @@ -8,7 +8,7 @@ //===----------------------------------------------------------------------===// // // This file implements the ScheduleDAG class, which is used as the common -// base class for SelectionDAG-based instruction scheduler. +// base class for instruction schedulers. // //===----------------------------------------------------------------------===// @@ -16,10 +16,9 @@ #define LLVM_CODEGEN_SCHEDULEDAG_H #include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/SelectionDAG.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/GraphTraits.h" -#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" namespace llvm { struct SUnit; @@ -31,51 +30,13 @@ namespace llvm { class TargetRegisterInfo; class ScheduleDAG; class SelectionDAG; - class SelectionDAGISel; + class SDNode; class TargetInstrInfo; class TargetInstrDesc; class TargetLowering; class TargetMachine; class TargetRegisterClass; - - /// HazardRecognizer - This determines whether or not an instruction can be - /// issued this cycle, and whether or not a noop needs to be inserted to handle - /// the hazard. - class HazardRecognizer { - public: - virtual ~HazardRecognizer(); - - enum HazardType { - NoHazard, // This instruction can be emitted at this cycle. - Hazard, // This instruction can't be emitted at this cycle. - NoopHazard // This instruction can't be emitted, and needs noops. - }; - - /// getHazardType - Return the hazard type of emitting this node. There are - /// three possible results. Either: - /// * NoHazard: it is legal to issue this instruction on this cycle. - /// * Hazard: issuing this instruction would stall the machine. If some - /// other instruction is available, issue it first. - /// * NoopHazard: issuing this instruction would break the program. If - /// some other instruction can be issued, do so, otherwise issue a noop. - virtual HazardType getHazardType(SDNode *) { - return NoHazard; - } - - /// EmitInstruction - This callback is invoked when an instruction is - /// emitted, to advance the hazard state. - virtual void EmitInstruction(SDNode *) {} - - /// AdvanceCycle - This callback is invoked when no instructions can be - /// issued on this cycle without a hazard. This should increment the - /// internal state of the hazard recognizer so that previously "Hazard" - /// instructions will now not be hazards. - virtual void AdvanceCycle() {} - - /// EmitNoop - This callback is invoked when a noop was added to the - /// instruction stream. - virtual void EmitNoop() {} - }; + template class GraphWriter; /// SDep - Scheduling dependency. It keeps track of dependent nodes, /// cost of the depdenency, etc. @@ -89,8 +50,7 @@ namespace llvm { : Dep(d), Reg(r), Cost(t), isCtrl(c), isSpecial(s) {} }; - /// SUnit - Scheduling unit. It's an wrapper around either a single SDNode or - /// a group of nodes flagged together. + /// SUnit - Scheduling unit. This is a node in the scheduling DAG. struct SUnit { private: SDNode *Node; // Representative node. @@ -294,12 +254,11 @@ namespace llvm { std::vector Sequence; // The schedule. Null SUnit*'s // represent noop instructions. std::vector SUnits; // The scheduling units. - SmallSet CommuteSet; // Nodes that should be commuted. ScheduleDAG(SelectionDAG *dag, MachineBasicBlock *bb, const TargetMachine &tm); - virtual ~ScheduleDAG() {} + virtual ~ScheduleDAG(); /// viewGraph - Pop up a GraphViz/gv window with the ScheduleDAG rendered /// using 'dot'. @@ -310,84 +269,27 @@ namespace llvm { /// void Run(); - /// isPassiveNode - Return true if the node is a non-scheduled leaf. + /// BuildSchedUnits - Build SUnits and set up their Preds and Succs + /// to form the scheduling dependency graph. /// - static bool isPassiveNode(SDNode *Node) { - if (isa(Node)) return true; - if (isa(Node)) return true; - if (isa(Node)) return true; - if (isa(Node)) return true; - if (isa(Node)) return true; - if (isa(Node)) return true; - if (isa(Node)) return true; - if (isa(Node)) return true; - if (isa(Node)) return true; - if (isa(Node)) return true; - if (Node->getOpcode() == ISD::EntryToken) return true; - return false; - } - - /// NewSUnit - Creates a new SUnit and return a ptr to it. - /// - SUnit *NewSUnit(SDNode *N) { - SUnits.push_back(SUnit(N, (unsigned)SUnits.size())); - SUnits.back().OrigNode = &SUnits.back(); - return &SUnits.back(); - } - - /// NewSUnit - Creates a new SUnit and return a ptr to it. - /// - SUnit *NewSUnit(MachineInstr *MI) { - SUnits.push_back(SUnit(MI, (unsigned)SUnits.size())); - SUnits.back().OrigNode = &SUnits.back(); - return &SUnits.back(); - } - - /// Clone - Creates a clone of the specified SUnit. It does not copy the - /// predecessors / successors info nor the temporary scheduling states. - SUnit *Clone(SUnit *N); - - /// BuildSchedUnits - Build SUnits from the selection dag that we are input. - /// This SUnit graph is similar to the SelectionDAG, but represents flagged - /// together nodes with a single SUnit. - void BuildSchedUnits(); + virtual void BuildSchedUnits() = 0; /// ComputeLatency - Compute node latency. /// - void ComputeLatency(SUnit *SU); + virtual void ComputeLatency(SUnit *SU) { SU->Latency = 1; } /// CalculateDepths, CalculateHeights - Calculate node depth / height. /// void CalculateDepths(); void CalculateHeights(); - /// CountResults - The results of target nodes have register or immediate - /// operands first, then an optional chain, and optional flag operands - /// (which do not go into the machine instrs.) - static unsigned CountResults(SDNode *Node); - - /// CountOperands - The inputs to target nodes have any actual inputs first, - /// followed by special operands that describe memory references, then an - /// optional chain operand, then flag operands. Compute the number of - /// actual operands that will go into the resulting MachineInstr. - static unsigned CountOperands(SDNode *Node); - - /// ComputeMemOperandsEnd - Find the index one past the last - /// MemOperandSDNode operand - static unsigned ComputeMemOperandsEnd(SDNode *Node); - - /// EmitNode - Generate machine code for an node and needed dependencies. - /// VRBaseMap contains, for each already emitted node, the first virtual - /// register number for the results of the node. - /// - void EmitNode(SDNode *Node, bool IsClone, - DenseMap &VRBaseMap); - + protected: /// EmitNoop - Emit a noop instruction. /// void EmitNoop(); - MachineBasicBlock *EmitSchedule(); + public: + virtual MachineBasicBlock *EmitSchedule() = 0; void dumpSchedule() const; @@ -396,41 +298,22 @@ namespace llvm { /// virtual void Schedule() = 0; - /// getGraphpNodeLabel - Return a label for an SUnit node in a Graphviz or similar - /// graph visualization. - virtual std::string getGraphNodeLabel(const SUnit *SU) const; + virtual void dumpNode(const SUnit *SU) const = 0; - private: - /// EmitSubregNode - Generate machine code for subreg nodes. - /// - void EmitSubregNode(SDNode *Node, - DenseMap &VRBaseMap); + /// getGraphNodeLabel - Return a label for an SUnit node in a visualization + /// of the ScheduleDAG. + virtual std::string getGraphNodeLabel(const SUnit *SU) const = 0; - /// getVR - Return the virtual register corresponding to the specified result - /// of the specified node. - unsigned getVR(SDValue Op, DenseMap &VRBaseMap); - - /// getDstOfCopyToRegUse - If the only use of the specified result number of - /// node is a CopyToReg, return its destination register. Return 0 otherwise. - unsigned getDstOfOnlyCopyToRegUse(SDNode *Node, unsigned ResNo) const; + /// addCustomGraphFeatures - Add custom features for a visualization of + /// the ScheduleDAG. + virtual void addCustomGraphFeatures(GraphWriter &GW) const {} - void AddOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, - const TargetInstrDesc *II, - DenseMap &VRBaseMap); + protected: void AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO); void EmitCrossRCCopy(SUnit *SU, DenseMap &VRBaseMap); - /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an - /// implicit physical register output. - void EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, - unsigned SrcReg, - DenseMap &VRBaseMap); - - void CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, - const TargetInstrDesc &II, - DenseMap &VRBaseMap); - + private: /// EmitLiveInCopy - Emit a copy for a live in physical register. If the /// physical register has only a single copy use, then coalesced the copy /// if possible. @@ -444,53 +327,8 @@ namespace llvm { /// and if it has live ins that need to be copied into vregs, emit the /// copies into the top of the block. void EmitLiveInCopies(MachineBasicBlock *MBB); - - /// BuildSchedUnitsFromMBB - Build SUnits from the MachineBasicBlock. - /// This SUnit graph is similar to the pre-regalloc SUnit graph, but represents - /// MachineInstrs directly instead of SDNodes. - void BuildSchedUnitsFromMBB(); }; - /// createBURRListDAGScheduler - This creates a bottom up register usage - /// reduction list scheduler. - ScheduleDAG* createBURRListDAGScheduler(SelectionDAGISel *IS, - SelectionDAG *DAG, - const TargetMachine *TM, - MachineBasicBlock *BB, - bool Fast); - - /// createTDRRListDAGScheduler - This creates a top down register usage - /// reduction list scheduler. - ScheduleDAG* createTDRRListDAGScheduler(SelectionDAGISel *IS, - SelectionDAG *DAG, - const TargetMachine *TM, - MachineBasicBlock *BB, - bool Fast); - - /// createTDListDAGScheduler - This creates a top-down list scheduler with - /// a hazard recognizer. - ScheduleDAG* createTDListDAGScheduler(SelectionDAGISel *IS, - SelectionDAG *DAG, - const TargetMachine *TM, - MachineBasicBlock *BB, - bool Fast); - - /// createFastDAGScheduler - This creates a "fast" scheduler. - /// - ScheduleDAG *createFastDAGScheduler(SelectionDAGISel *IS, - SelectionDAG *DAG, - const TargetMachine *TM, - MachineBasicBlock *BB, - bool Fast); - - /// createDefaultScheduler - This creates an instruction scheduler appropriate - /// for the target. - ScheduleDAG* createDefaultScheduler(SelectionDAGISel *IS, - SelectionDAG *DAG, - const TargetMachine *TM, - MachineBasicBlock *BB, - bool Fast); - class SUnitIterator : public forward_iterator { SUnit *Node; unsigned Operand; diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h new file mode 100644 index 00000000000..703c5b0876e --- /dev/null +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -0,0 +1,69 @@ +//==- llvm/CodeGen/ScheduleDAGInstrs.h - MachineInstr Scheduling -*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the ScheduleDAGInstrs class, which implements +// scheduling for a MachineInstr-based dependency graph. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_SCHEDULEDAGINSTRS_H +#define LLVM_CODEGEN_SCHEDULEDAGINSTRS_H + +#include "llvm/CodeGen/ScheduleDAG.h" + +namespace llvm { + struct SUnit; + class MachineConstantPool; + class MachineFunction; + class MachineModuleInfo; + class MachineRegisterInfo; + class MachineInstr; + class TargetRegisterInfo; + class ScheduleDAG; + class SelectionDAG; + class SelectionDAGISel; + class TargetInstrInfo; + class TargetInstrDesc; + class TargetLowering; + class TargetMachine; + class TargetRegisterClass; + + class ScheduleDAGInstrs : public ScheduleDAG { + public: + ScheduleDAGInstrs(MachineBasicBlock *bb, + const TargetMachine &tm); + + virtual ~ScheduleDAGInstrs() {} + + /// NewSUnit - Creates a new SUnit and return a ptr to it. + /// + SUnit *NewSUnit(MachineInstr *MI) { + SUnits.push_back(SUnit(MI, (unsigned)SUnits.size())); + SUnits.back().OrigNode = &SUnits.back(); + return &SUnits.back(); + } + + /// BuildSchedUnits - Build SUnits from the MachineBasicBlock that we are + /// input. + virtual void BuildSchedUnits(); + + virtual MachineBasicBlock *EmitSchedule(); + + /// Schedule - Order nodes according to selected style, filling + /// in the Sequence member. + /// + virtual void Schedule() = 0; + + virtual void dumpNode(const SUnit *SU) const; + + virtual std::string getGraphNodeLabel(const SUnit *SU) const; + }; +} + +#endif diff --git a/include/llvm/CodeGen/ScheduleDAGSDNodes.h b/include/llvm/CodeGen/ScheduleDAGSDNodes.h new file mode 100644 index 00000000000..caa1d0a281c --- /dev/null +++ b/include/llvm/CodeGen/ScheduleDAGSDNodes.h @@ -0,0 +1,233 @@ +//===---- llvm/CodeGen/ScheduleDAGSDNodes.h - SDNode Scheduling -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the ScheduleDAGSDNodes class, which implements +// scheduling for an SDNode-based dependency graph. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_SCHEDULEDAGSDNODES_H +#define LLVM_CODEGEN_SCHEDULEDAGSDNODES_H + +#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/ADT/SmallSet.h" + +namespace llvm { + struct SUnit; + class MachineConstantPool; + class MachineFunction; + class MachineModuleInfo; + class MachineRegisterInfo; + class MachineInstr; + class TargetRegisterInfo; + class ScheduleDAG; + class SelectionDAG; + class SelectionDAGISel; + class TargetInstrInfo; + class TargetInstrDesc; + class TargetLowering; + class TargetMachine; + class TargetRegisterClass; + + /// HazardRecognizer - This determines whether or not an instruction can be + /// issued this cycle, and whether or not a noop needs to be inserted to handle + /// the hazard. + class HazardRecognizer { + public: + virtual ~HazardRecognizer(); + + enum HazardType { + NoHazard, // This instruction can be emitted at this cycle. + Hazard, // This instruction can't be emitted at this cycle. + NoopHazard // This instruction can't be emitted, and needs noops. + }; + + /// getHazardType - Return the hazard type of emitting this node. There are + /// three possible results. Either: + /// * NoHazard: it is legal to issue this instruction on this cycle. + /// * Hazard: issuing this instruction would stall the machine. If some + /// other instruction is available, issue it first. + /// * NoopHazard: issuing this instruction would break the program. If + /// some other instruction can be issued, do so, otherwise issue a noop. + virtual HazardType getHazardType(SDNode *) { + return NoHazard; + } + + /// EmitInstruction - This callback is invoked when an instruction is + /// emitted, to advance the hazard state. + virtual void EmitInstruction(SDNode *) {} + + /// AdvanceCycle - This callback is invoked when no instructions can be + /// issued on this cycle without a hazard. This should increment the + /// internal state of the hazard recognizer so that previously "Hazard" + /// instructions will now not be hazards. + virtual void AdvanceCycle() {} + + /// EmitNoop - This callback is invoked when a noop was added to the + /// instruction stream. + virtual void EmitNoop() {} + }; + + class ScheduleDAGSDNodes : public ScheduleDAG { + public: + SmallSet CommuteSet; // Nodes that should be commuted. + + ScheduleDAGSDNodes(SelectionDAG *dag, MachineBasicBlock *bb, + const TargetMachine &tm); + + virtual ~ScheduleDAGSDNodes() {} + + /// isPassiveNode - Return true if the node is a non-scheduled leaf. + /// + static bool isPassiveNode(SDNode *Node) { + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (Node->getOpcode() == ISD::EntryToken) return true; + return false; + } + + /// NewSUnit - Creates a new SUnit and return a ptr to it. + /// + SUnit *NewSUnit(SDNode *N) { + SUnits.push_back(SUnit(N, (unsigned)SUnits.size())); + SUnits.back().OrigNode = &SUnits.back(); + return &SUnits.back(); + } + + /// Clone - Creates a clone of the specified SUnit. It does not copy the + /// predecessors / successors info nor the temporary scheduling states. + /// + SUnit *Clone(SUnit *N); + + virtual SelectionDAG *getDAG() { return DAG; } + + /// BuildSchedUnits - Build SUnits from the selection dag that we are input. + /// This SUnit graph is similar to the SelectionDAG, but represents flagged + /// together nodes with a single SUnit. + virtual void BuildSchedUnits(); + + /// ComputeLatency - Compute node latency. + /// + virtual void ComputeLatency(SUnit *SU); + + /// CountResults - The results of target nodes have register or immediate + /// operands first, then an optional chain, and optional flag operands + /// (which do not go into the machine instrs.) + static unsigned CountResults(SDNode *Node); + + /// CountOperands - The inputs to target nodes have any actual inputs first, + /// followed by special operands that describe memory references, then an + /// optional chain operand, then flag operands. Compute the number of + /// actual operands that will go into the resulting MachineInstr. + static unsigned CountOperands(SDNode *Node); + + /// ComputeMemOperandsEnd - Find the index one past the last + /// MemOperandSDNode operand + static unsigned ComputeMemOperandsEnd(SDNode *Node); + + /// EmitNode - Generate machine code for an node and needed dependencies. + /// VRBaseMap contains, for each already emitted node, the first virtual + /// register number for the results of the node. + /// + void EmitNode(SDNode *Node, bool IsClone, + DenseMap &VRBaseMap); + + virtual MachineBasicBlock *EmitSchedule(); + + /// Schedule - Order nodes according to selected style, filling + /// in the Sequence member. + /// + virtual void Schedule() = 0; + + virtual void dumpNode(const SUnit *SU) const; + + virtual std::string getGraphNodeLabel(const SUnit *SU) const; + + virtual void getCustomGraphFeatures(GraphWriter &GW) const; + + private: + /// EmitSubregNode - Generate machine code for subreg nodes. + /// + void EmitSubregNode(SDNode *Node, + DenseMap &VRBaseMap); + + /// getVR - Return the virtual register corresponding to the specified result + /// of the specified node. + unsigned getVR(SDValue Op, DenseMap &VRBaseMap); + + /// getDstOfCopyToRegUse - If the only use of the specified result number of + /// node is a CopyToReg, return its destination register. Return 0 otherwise. + unsigned getDstOfOnlyCopyToRegUse(SDNode *Node, unsigned ResNo) const; + + void AddOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, + const TargetInstrDesc *II, + DenseMap &VRBaseMap); + + /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an + /// implicit physical register output. + void EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, + unsigned SrcReg, + DenseMap &VRBaseMap); + + void CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, + const TargetInstrDesc &II, + DenseMap &VRBaseMap); + }; + + /// createBURRListDAGScheduler - This creates a bottom up register usage + /// reduction list scheduler. + ScheduleDAG* createBURRListDAGScheduler(SelectionDAGISel *IS, + SelectionDAG *DAG, + const TargetMachine *TM, + MachineBasicBlock *BB, + bool Fast); + + /// createTDRRListDAGScheduler - This creates a top down register usage + /// reduction list scheduler. + ScheduleDAG* createTDRRListDAGScheduler(SelectionDAGISel *IS, + SelectionDAG *DAG, + const TargetMachine *TM, + MachineBasicBlock *BB, + bool Fast); + + /// createTDListDAGScheduler - This creates a top-down list scheduler with + /// a hazard recognizer. + ScheduleDAG* createTDListDAGScheduler(SelectionDAGISel *IS, + SelectionDAG *DAG, + const TargetMachine *TM, + MachineBasicBlock *BB, + bool Fast); + + /// createFastDAGScheduler - This creates a "fast" scheduler. + /// + ScheduleDAG *createFastDAGScheduler(SelectionDAGISel *IS, + SelectionDAG *DAG, + const TargetMachine *TM, + MachineBasicBlock *BB, + bool Fast); + + /// createDefaultScheduler - This creates an instruction scheduler appropriate + /// for the target. + ScheduleDAG* createDefaultScheduler(SelectionDAGISel *IS, + SelectionDAG *DAG, + const TargetMachine *TM, + MachineBasicBlock *BB, + bool Fast); +} + +#endif diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index a7aad33403a..fed323742c4 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -8,6 +8,7 @@ add_llvm_library(LLVMCodeGen IfConversion.cpp IntrinsicLowering.cpp LLVMTargetMachine.cpp + LatencyPriorityQueue.cpp LiveInterval.cpp LiveIntervalAnalysis.cpp LiveStackAnalysis.cpp @@ -40,6 +41,10 @@ add_llvm_library(LLVMCodeGen RegAllocSimple.cpp RegisterCoalescer.cpp RegisterScavenging.cpp + ScheduleDAG.cpp + ScheduleDAGEmit.cpp + ScheduleDAGInstrs.cpp + ScheduleDAGPrinter.cpp ShadowStackGC.cpp SimpleRegisterCoalescing.cpp StackProtector.cpp diff --git a/lib/CodeGen/LatencyPriorityQueue.cpp b/lib/CodeGen/LatencyPriorityQueue.cpp new file mode 100644 index 00000000000..70b6574996b --- /dev/null +++ b/lib/CodeGen/LatencyPriorityQueue.cpp @@ -0,0 +1,165 @@ +//===---- LatencyPriorityQueue.cpp - A latency-oriented priority queue ----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the LatencyPriorityQueue class, which is a +// SchedulingPriorityQueue that schedules using latency information to +// reduce the length of the critical path through the basic block. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "scheduler" +#include "llvm/CodeGen/LatencyPriorityQueue.h" +#include "llvm/Support/Debug.h" +using namespace llvm; + +bool latency_sort::operator()(const SUnit *LHS, const SUnit *RHS) const { + unsigned LHSNum = LHS->NodeNum; + unsigned RHSNum = RHS->NodeNum; + + // The most important heuristic is scheduling the critical path. + unsigned LHSLatency = PQ->getLatency(LHSNum); + unsigned RHSLatency = PQ->getLatency(RHSNum); + if (LHSLatency < RHSLatency) return true; + if (LHSLatency > RHSLatency) return false; + + // After that, if two nodes have identical latencies, look to see if one will + // unblock more other nodes than the other. + unsigned LHSBlocked = PQ->getNumSolelyBlockNodes(LHSNum); + unsigned RHSBlocked = PQ->getNumSolelyBlockNodes(RHSNum); + if (LHSBlocked < RHSBlocked) return true; + if (LHSBlocked > RHSBlocked) return false; + + // Finally, just to provide a stable ordering, use the node number as a + // deciding factor. + return LHSNum < RHSNum; +} + + +/// CalcNodePriority - Calculate the maximal path from the node to the exit. +/// +int LatencyPriorityQueue::CalcLatency(const SUnit &SU) { + int &Latency = Latencies[SU.NodeNum]; + if (Latency != -1) + return Latency; + + std::vector WorkList; + WorkList.push_back(&SU); + while (!WorkList.empty()) { + const SUnit *Cur = WorkList.back(); + bool AllDone = true; + int MaxSuccLatency = 0; + for (SUnit::const_succ_iterator I = Cur->Succs.begin(),E = Cur->Succs.end(); + I != E; ++I) { + int SuccLatency = Latencies[I->Dep->NodeNum]; + if (SuccLatency == -1) { + AllDone = false; + WorkList.push_back(I->Dep); + } else { + MaxSuccLatency = std::max(MaxSuccLatency, SuccLatency); + } + } + if (AllDone) { + Latencies[Cur->NodeNum] = MaxSuccLatency + Cur->Latency; + WorkList.pop_back(); + } + } + + return Latency; +} + +/// CalculatePriorities - Calculate priorities of all scheduling units. +void LatencyPriorityQueue::CalculatePriorities() { + Latencies.assign(SUnits->size(), -1); + NumNodesSolelyBlocking.assign(SUnits->size(), 0); + + // For each node, calculate the maximal path from the node to the exit. + std::vector > WorkList; + for (unsigned i = 0, e = SUnits->size(); i != e; ++i) { + const SUnit *SU = &(*SUnits)[i]; + if (SU->Succs.empty()) + WorkList.push_back(std::make_pair(SU, 0U)); + } + + while (!WorkList.empty()) { + const SUnit *SU = WorkList.back().first; + unsigned SuccLat = WorkList.back().second; + WorkList.pop_back(); + int &Latency = Latencies[SU->NodeNum]; + if (Latency == -1 || (SU->Latency + SuccLat) > (unsigned)Latency) { + Latency = SU->Latency + SuccLat; + for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); + I != E; ++I) + WorkList.push_back(std::make_pair(I->Dep, Latency)); + } + } +} + +/// getSingleUnscheduledPred - If there is exactly one unscheduled predecessor +/// of SU, return it, otherwise return null. +SUnit *LatencyPriorityQueue::getSingleUnscheduledPred(SUnit *SU) { + SUnit *OnlyAvailablePred = 0; + for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); + I != E; ++I) { + SUnit &Pred = *I->Dep; + if (!Pred.isScheduled) { + // We found an available, but not scheduled, predecessor. If it's the + // only one we have found, keep track of it... otherwise give up. + if (OnlyAvailablePred && OnlyAvailablePred != &Pred) + return 0; + OnlyAvailablePred = &Pred; + } + } + + return OnlyAvailablePred; +} + +void LatencyPriorityQueue::push_impl(SUnit *SU) { + // Look at all of the successors of this node. Count the number of nodes that + // this node is the sole unscheduled node for. + unsigned NumNodesBlocking = 0; + for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); + I != E; ++I) + if (getSingleUnscheduledPred(I->Dep) == SU) + ++NumNodesBlocking; + NumNodesSolelyBlocking[SU->NodeNum] = NumNodesBlocking; + + Queue.push(SU); +} + + +// ScheduledNode - As nodes are scheduled, we look to see if there are any +// successor nodes that have a single unscheduled predecessor. If so, that +// single predecessor has a higher priority, since scheduling it will make +// the node available. +void LatencyPriorityQueue::ScheduledNode(SUnit *SU) { + for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); + I != E; ++I) + AdjustPriorityOfUnscheduledPreds(I->Dep); +} + +/// AdjustPriorityOfUnscheduledPreds - One of the predecessors of SU was just +/// scheduled. If SU is not itself available, then there is at least one +/// predecessor node that has not been scheduled yet. If SU has exactly ONE +/// unscheduled predecessor, we want to increase its priority: it getting +/// scheduled will make this node available, so it is better than some other +/// node of the same priority that will not make a node available. +void LatencyPriorityQueue::AdjustPriorityOfUnscheduledPreds(SUnit *SU) { + if (SU->isAvailable) return; // All preds scheduled. + + SUnit *OnlyAvailablePred = getSingleUnscheduledPred(SU); + if (OnlyAvailablePred == 0 || !OnlyAvailablePred->isAvailable) return; + + // Okay, we found a single predecessor that is available, but not scheduled. + // Since it is available, it must be in the priority queue. First remove it. + remove(OnlyAvailablePred); + + // Reinsert the node into the priority queue, which recomputes its + // NumNodesSolelyBlocking value. + push(OnlyAvailablePred); +} diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp index 41981d3ff14..3a31eaa09b6 100644 --- a/lib/CodeGen/PostRASchedulerList.cpp +++ b/lib/CodeGen/PostRASchedulerList.cpp @@ -20,47 +20,234 @@ #define DEBUG_TYPE "post-RA-sched" #include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/ScheduleDAGInstrs.h" +#include "llvm/CodeGen/LatencyPriorityQueue.h" +#include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" +#include "llvm/ADT/Statistic.h" using namespace llvm; +STATISTIC(NumStalls, "Number of pipeline stalls"); + namespace { - class VISIBILITY_HIDDEN SchedulePostRATDList : public MachineFunctionPass { + class VISIBILITY_HIDDEN PostRAScheduler : public MachineFunctionPass { public: static char ID; - SchedulePostRATDList() : MachineFunctionPass(&ID) {} + PostRAScheduler() : MachineFunctionPass(&ID) {} + private: + MachineFunction *MF; + const TargetMachine *TM; + public: + const char *getPassName() const { + return "Post RA top-down list latency scheduler (STUB)"; + } + + bool runOnMachineFunction(MachineFunction &Fn); + }; + char PostRAScheduler::ID = 0; + + class VISIBILITY_HIDDEN SchedulePostRATDList : public ScheduleDAGInstrs { + public: + SchedulePostRATDList(MachineBasicBlock *mbb, const TargetMachine &tm) + : ScheduleDAGInstrs(mbb, tm) {} private: MachineFunction *MF; const TargetMachine *TM; + + /// AvailableQueue - The priority queue to use for the available SUnits. + /// + LatencyPriorityQueue AvailableQueue; + + /// PendingQueue - This contains all of the instructions whose operands have + /// been issued, but their results are not ready yet (due to the latency of + /// the operation). Once the operands becomes available, the instruction is + /// added to the AvailableQueue. + std::vector PendingQueue; + public: const char *getPassName() const { return "Post RA top-down list latency scheduler (STUB)"; } bool runOnMachineFunction(MachineFunction &Fn); + + void Schedule(); + + private: + void ReleaseSucc(SUnit *SU, SUnit *SuccSU, bool isChain); + void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle); + void ListScheduleTopDown(); }; - char SchedulePostRATDList::ID = 0; } -bool SchedulePostRATDList::runOnMachineFunction(MachineFunction &Fn) { - DOUT << "SchedulePostRATDList\n"; +bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { + DOUT << "PostRAScheduler\n"; MF = &Fn; TM = &MF->getTarget(); // Loop over all of the basic blocks for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end(); - MBB != MBBe; ++MBB) - ; + MBB != MBBe; ++MBB) { + + SchedulePostRATDList Scheduler(MBB, *TM); + + Scheduler.Run(); + + Scheduler.EmitSchedule(); + } return true; } +/// Schedule - Schedule the DAG using list scheduling. +void SchedulePostRATDList::Schedule() { + DOUT << "********** List Scheduling **********\n"; + + // Build scheduling units. + BuildSchedUnits(); + + AvailableQueue.initNodes(SUnits); + + ListScheduleTopDown(); + + AvailableQueue.releaseState(); +} + +//===----------------------------------------------------------------------===// +// Top-Down Scheduling +//===----------------------------------------------------------------------===// + +/// ReleaseSucc - Decrement the NumPredsLeft count of a successor. Add it to +/// the PendingQueue if the count reaches zero. Also update its cycle bound. +void SchedulePostRATDList::ReleaseSucc(SUnit *SU, SUnit *SuccSU, bool isChain) { + --SuccSU->NumPredsLeft; + +#ifndef NDEBUG + if (SuccSU->NumPredsLeft < 0) { + cerr << "*** Scheduling failed! ***\n"; + SuccSU->dump(this); + cerr << " has been released too many times!\n"; + assert(0); + } +#endif + + // Compute how many cycles it will be before this actually becomes + // available. This is the max of the start time of all predecessors plus + // their latencies. + // If this is a token edge, we don't need to wait for the latency of the + // preceeding instruction (e.g. a long-latency load) unless there is also + // some other data dependence. + unsigned PredDoneCycle = SU->Cycle; + if (!isChain) + PredDoneCycle += SU->Latency; + else if (SU->Latency) + PredDoneCycle += 1; + SuccSU->CycleBound = std::max(SuccSU->CycleBound, PredDoneCycle); + + if (SuccSU->NumPredsLeft == 0) { + PendingQueue.push_back(SuccSU); + } +} + +/// ScheduleNodeTopDown - Add the node to the schedule. Decrement the pending +/// count of its successors. If a successor pending count is zero, add it to +/// the Available queue. +void SchedulePostRATDList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle) { + DOUT << "*** Scheduling [" << CurCycle << "]: "; + DEBUG(SU->dump(this)); + + Sequence.push_back(SU); + SU->Cycle = CurCycle; + + // Top down: release successors. + for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); + I != E; ++I) + ReleaseSucc(SU, I->Dep, I->isCtrl); + + SU->isScheduled = true; + AvailableQueue.ScheduledNode(SU); +} + +/// ListScheduleTopDown - The main loop of list scheduling for top-down +/// schedulers. +void SchedulePostRATDList::ListScheduleTopDown() { + unsigned CurCycle = 0; + + // All leaves to Available queue. + for (unsigned i = 0, e = SUnits.size(); i != e; ++i) { + // It is available if it has no predecessors. + if (SUnits[i].Preds.empty()) { + AvailableQueue.push(&SUnits[i]); + SUnits[i].isAvailable = true; + } + } + + // While Available queue is not empty, grab the node with the highest + // priority. If it is not ready put it back. Schedule the node. + Sequence.reserve(SUnits.size()); + while (!AvailableQueue.empty() || !PendingQueue.empty()) { + // Check to see if any of the pending instructions are ready to issue. If + // so, add them to the available queue. + for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) { + if (PendingQueue[i]->CycleBound == CurCycle) { + AvailableQueue.push(PendingQueue[i]); + PendingQueue[i]->isAvailable = true; + PendingQueue[i] = PendingQueue.back(); + PendingQueue.pop_back(); + --i; --e; + } else { + assert(PendingQueue[i]->CycleBound > CurCycle && "Negative latency?"); + } + } + + // If there are no instructions available, don't try to issue anything, and + // don't advance the hazard recognizer. + if (AvailableQueue.empty()) { + ++CurCycle; + continue; + } + + SUnit *FoundSUnit = AvailableQueue.pop(); + + // If we found a node to schedule, do it now. + if (FoundSUnit) { + ScheduleNodeTopDown(FoundSUnit, CurCycle); + + // If this is a pseudo-op node, we don't want to increment the current + // cycle. + if (FoundSUnit->Latency) // Don't increment CurCycle for pseudo-ops! + ++CurCycle; + } else { + // Otherwise, we have a pipeline stall, but no other problem, just advance + // the current cycle and try again. + DOUT << "*** Advancing cycle, no work to do\n"; + ++NumStalls; + ++CurCycle; + } + } + +#ifndef NDEBUG + // Verify that all SUnits were scheduled. + bool AnyNotSched = false; + for (unsigned i = 0, e = SUnits.size(); i != e; ++i) { + if (SUnits[i].NumPredsLeft != 0) { + if (!AnyNotSched) + cerr << "*** List scheduling failed! ***\n"; + SUnits[i].dump(this); + cerr << "has not been scheduled!\n"; + AnyNotSched = true; + } + } + assert(!AnyNotSched); +#endif +} //===----------------------------------------------------------------------===// // Public Constructor Functions //===----------------------------------------------------------------------===// FunctionPass *llvm::createPostRAScheduler() { - return new SchedulePostRATDList(); + return new PostRAScheduler(); } diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp new file mode 100644 index 00000000000..e6a8d13c2f5 --- /dev/null +++ b/lib/CodeGen/ScheduleDAG.cpp @@ -0,0 +1,210 @@ +//===---- ScheduleDAG.cpp - Implement the ScheduleDAG class ---------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements the ScheduleDAG class, which is a base class used by +// scheduling implementation classes. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "pre-RA-sched" +#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Support/Debug.h" +using namespace llvm; + +ScheduleDAG::ScheduleDAG(SelectionDAG *dag, MachineBasicBlock *bb, + const TargetMachine &tm) + : DAG(dag), BB(bb), TM(tm), MRI(BB->getParent()->getRegInfo()) { + TII = TM.getInstrInfo(); + MF = BB->getParent(); + TRI = TM.getRegisterInfo(); + TLI = TM.getTargetLowering(); + ConstPool = MF->getConstantPool(); +} + +ScheduleDAG::~ScheduleDAG() {} + +/// CalculateDepths - compute depths using algorithms for the longest +/// paths in the DAG +void ScheduleDAG::CalculateDepths() { + unsigned DAGSize = SUnits.size(); + std::vector WorkList; + WorkList.reserve(DAGSize); + + // Initialize the data structures + for (unsigned i = 0, e = DAGSize; i != e; ++i) { + SUnit *SU = &SUnits[i]; + unsigned Degree = SU->Preds.size(); + // Temporarily use the Depth field as scratch space for the degree count. + SU->Depth = Degree; + + // Is it a node without dependencies? + if (Degree == 0) { + assert(SU->Preds.empty() && "SUnit should have no predecessors"); + // Collect leaf nodes + WorkList.push_back(SU); + } + } + + // Process nodes in the topological order + while (!WorkList.empty()) { + SUnit *SU = WorkList.back(); + WorkList.pop_back(); + unsigned SUDepth = 0; + + // Use dynamic programming: + // When current node is being processed, all of its dependencies + // are already processed. + // So, just iterate over all predecessors and take the longest path + for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); + I != E; ++I) { + unsigned PredDepth = I->Dep->Depth; + if (PredDepth+1 > SUDepth) { + SUDepth = PredDepth + 1; + } + } + + SU->Depth = SUDepth; + + // Update degrees of all nodes depending on current SUnit + for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); + I != E; ++I) { + SUnit *SU = I->Dep; + if (!--SU->Depth) + // If all dependencies of the node are processed already, + // then the longest path for the node can be computed now + WorkList.push_back(SU); + } + } +} + +/// CalculateHeights - compute heights using algorithms for the longest +/// paths in the DAG +void ScheduleDAG::CalculateHeights() { + unsigned DAGSize = SUnits.size(); + std::vector WorkList; + WorkList.reserve(DAGSize); + + // Initialize the data structures + for (unsigned i = 0, e = DAGSize; i != e; ++i) { + SUnit *SU = &SUnits[i]; + unsigned Degree = SU->Succs.size(); + // Temporarily use the Height field as scratch space for the degree count. + SU->Height = Degree; + + // Is it a node without dependencies? + if (Degree == 0) { + assert(SU->Succs.empty() && "Something wrong"); + assert(WorkList.empty() && "Should be empty"); + // Collect leaf nodes + WorkList.push_back(SU); + } + } + + // Process nodes in the topological order + while (!WorkList.empty()) { + SUnit *SU = WorkList.back(); + WorkList.pop_back(); + unsigned SUHeight = 0; + + // Use dynamic programming: + // When current node is being processed, all of its dependencies + // are already processed. + // So, just iterate over all successors and take the longest path + for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); + I != E; ++I) { + unsigned SuccHeight = I->Dep->Height; + if (SuccHeight+1 > SUHeight) { + SUHeight = SuccHeight + 1; + } + } + + SU->Height = SUHeight; + + // Update degrees of all nodes depending on current SUnit + for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); + I != E; ++I) { + SUnit *SU = I->Dep; + if (!--SU->Height) + // If all dependencies of the node are processed already, + // then the longest path for the node can be computed now + WorkList.push_back(SU); + } + } +} + +/// dump - dump the schedule. +void ScheduleDAG::dumpSchedule() const { + for (unsigned i = 0, e = Sequence.size(); i != e; i++) { + if (SUnit *SU = Sequence[i]) + SU->dump(this); + else + cerr << "**** NOOP ****\n"; + } +} + + +/// Run - perform scheduling. +/// +void ScheduleDAG::Run() { + Schedule(); + + DOUT << "*** Final schedule ***\n"; + DEBUG(dumpSchedule()); + DOUT << "\n"; +} + +/// SUnit - Scheduling unit. It's an wrapper around either a single SDNode or +/// a group of nodes flagged together. +void SUnit::dump(const ScheduleDAG *G) const { + cerr << "SU(" << NodeNum << "): "; + G->dumpNode(this); +} + +void SUnit::dumpAll(const ScheduleDAG *G) const { + dump(G); + + cerr << " # preds left : " << NumPredsLeft << "\n"; + cerr << " # succs left : " << NumSuccsLeft << "\n"; + cerr << " Latency : " << Latency << "\n"; + cerr << " Depth : " << Depth << "\n"; + cerr << " Height : " << Height << "\n"; + + if (Preds.size() != 0) { + cerr << " Predecessors:\n"; + for (SUnit::const_succ_iterator I = Preds.begin(), E = Preds.end(); + I != E; ++I) { + if (I->isCtrl) + cerr << " ch #"; + else + cerr << " val #"; + cerr << I->Dep << " - SU(" << I->Dep->NodeNum << ")"; + if (I->isSpecial) + cerr << " *"; + cerr << "\n"; + } + } + if (Succs.size() != 0) { + cerr << " Successors:\n"; + for (SUnit::const_succ_iterator I = Succs.begin(), E = Succs.end(); + I != E; ++I) { + if (I->isCtrl) + cerr << " ch #"; + else + cerr << " val #"; + cerr << I->Dep << " - SU(" << I->Dep->NodeNum << ")"; + if (I->isSpecial) + cerr << " *"; + cerr << "\n"; + } + } + cerr << "\n"; +} diff --git a/lib/CodeGen/ScheduleDAGEmit.cpp b/lib/CodeGen/ScheduleDAGEmit.cpp new file mode 100644 index 00000000000..ce3283dc3df --- /dev/null +++ b/lib/CodeGen/ScheduleDAGEmit.cpp @@ -0,0 +1,72 @@ +//===---- ScheduleDAGEmit.cpp - Emit routines for the ScheduleDAG class ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements the Emit routines for the ScheduleDAG class, which creates +// MachineInstrs according to the computed schedule. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "pre-RA-sched" +#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/MachineConstantPool.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/Target/TargetData.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" +using namespace llvm; + +void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO) { + MI->addMemOperand(*MF, MO); +} + +void ScheduleDAG::EmitNoop() { + TII->insertNoop(*BB, BB->end()); +} + +void ScheduleDAG::EmitCrossRCCopy(SUnit *SU, + DenseMap &VRBaseMap) { + for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); + I != E; ++I) { + if (I->isCtrl) continue; // ignore chain preds + if (I->Dep->CopyDstRC) { + // Copy to physical register. + DenseMap::iterator VRI = VRBaseMap.find(I->Dep); + assert(VRI != VRBaseMap.end() && "Node emitted out of order - late"); + // Find the destination physical register. + unsigned Reg = 0; + for (SUnit::const_succ_iterator II = SU->Succs.begin(), + EE = SU->Succs.end(); II != EE; ++II) { + if (I->Reg) { + Reg = I->Reg; + break; + } + } + assert(I->Reg && "Unknown physical register!"); + TII->copyRegToReg(*BB, BB->end(), Reg, VRI->second, + SU->CopyDstRC, SU->CopySrcRC); + } else { + // Copy from physical register. + assert(I->Reg && "Unknown physical register!"); + unsigned VRBase = MRI.createVirtualRegister(SU->CopyDstRC); + bool isNew = VRBaseMap.insert(std::make_pair(SU, VRBase)).second; + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); + TII->copyRegToReg(*BB, BB->end(), VRBase, I->Reg, + SU->CopyDstRC, SU->CopySrcRC); + } + break; + } +} diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp new file mode 100644 index 00000000000..72008c9c95f --- /dev/null +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -0,0 +1,135 @@ +//===---- ScheduleDAG.cpp - Implement the ScheduleDAG class ---------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements the ScheduleDAG class, which is a base class used by +// scheduling implementation classes. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "sched-instrs" +#include "llvm/CodeGen/ScheduleDAGInstrs.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +using namespace llvm; + +ScheduleDAGInstrs::ScheduleDAGInstrs(MachineBasicBlock *bb, + const TargetMachine &tm) + : ScheduleDAG(0, bb, tm) {} + +void ScheduleDAGInstrs::BuildSchedUnits() { + SUnits.clear(); + SUnits.reserve(BB->size()); + + std::vector PendingLoads; + SUnit *Terminator = 0; + SUnit *Chain = 0; + SUnit *Defs[TargetRegisterInfo::FirstVirtualRegister] = {}; + std::vector Uses[TargetRegisterInfo::FirstVirtualRegister] = {}; + int Cost = 1; // FIXME + + for (MachineBasicBlock::iterator MII = BB->end(), MIE = BB->begin(); + MII != MIE; --MII) { + MachineInstr *MI = prior(MII); + SUnit *SU = NewSUnit(MI); + + for (unsigned j = 0, n = MI->getNumOperands(); j != n; ++j) { + const MachineOperand &MO = MI->getOperand(j); + if (!MO.isReg()) continue; + unsigned Reg = MO.getReg(); + if (Reg == 0) continue; + + assert(TRI->isPhysicalRegister(Reg) && "Virtual register encountered!"); + std::vector &UseList = Uses[Reg]; + SUnit *&Def = Defs[Reg]; + // Optionally add output and anti dependences + if (Def && Def != SU) + Def->addPred(SU, /*isCtrl=*/true, /*isSpecial=*/false, + /*PhyReg=*/Reg, Cost); + for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) { + SUnit *&Def = Defs[*Alias]; + if (Def && Def != SU) + Def->addPred(SU, /*isCtrl=*/true, /*isSpecial=*/false, + /*PhyReg=*/*Alias, Cost); + } + + if (MO.isDef()) { + // Add any data dependencies. + for (unsigned i = 0, e = UseList.size(); i != e; ++i) + if (UseList[i] != SU) + UseList[i]->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false, + /*PhysReg=*/Reg, Cost); + for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) { + std::vector &UseList = Uses[*Alias]; + for (unsigned i = 0, e = UseList.size(); i != e; ++i) + if (UseList[i] != SU) + UseList[i]->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false, + /*PhysReg=*/*Alias, Cost); + } + + UseList.clear(); + Def = SU; + } else { + UseList.push_back(SU); + } + } + bool False = false; + bool True = true; + if (!MI->isSafeToMove(TII, False)) { + if (Chain) + Chain->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false); + for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) + PendingLoads[k]->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false); + PendingLoads.clear(); + Chain = SU; + } else if (!MI->isSafeToMove(TII, True)) { + if (Chain) + Chain->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false); + PendingLoads.push_back(SU); + } + if (Terminator && SU->Succs.empty()) + Terminator->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false); + if (MI->getDesc().isTerminator()) + Terminator = SU; + } +} + +void ScheduleDAGInstrs::dumpNode(const SUnit *SU) const { + SU->getInstr()->dump(); +} + +std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const { + std::string s; + raw_string_ostream oss(s); + SU->getInstr()->print(oss); + return oss.str(); +} + +// EmitSchedule - Emit the machine code in scheduled order. +MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() { + // For MachineInstr-based scheduling, we're rescheduling the instructions in + // the block, so start by removing them from the block. + while (!BB->empty()) + BB->remove(BB->begin()); + + for (unsigned i = 0, e = Sequence.size(); i != e; i++) { + SUnit *SU = Sequence[i]; + if (!SU) { + // Null SUnit* is a noop. + EmitNoop(); + continue; + } + + BB->push_back(SU->getInstr()); + } + + return BB; +} diff --git a/lib/CodeGen/ScheduleDAGPrinter.cpp b/lib/CodeGen/ScheduleDAGPrinter.cpp new file mode 100644 index 00000000000..b5d030c1b81 --- /dev/null +++ b/lib/CodeGen/ScheduleDAGPrinter.cpp @@ -0,0 +1,93 @@ +//===-- ScheduleDAGPrinter.cpp - Implement ScheduleDAG::viewGraph() -------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements the ScheduleDAG::viewGraph method. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Constants.h" +#include "llvm/Function.h" +#include "llvm/Assembly/Writer.h" +#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/MachineConstantPool.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/GraphWriter.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Config/config.h" +#include +using namespace llvm; + +namespace llvm { + template<> + struct DOTGraphTraits : public DefaultDOTGraphTraits { + static std::string getGraphName(const ScheduleDAG *G) { + return G->MF->getFunction()->getName(); + } + + static bool renderGraphFromBottomUp() { + return true; + } + + static bool hasNodeAddressLabel(const SUnit *Node, + const ScheduleDAG *Graph) { + return true; + } + + /// If you want to override the dot attributes printed for a particular + /// edge, override this method. + template + static std::string getEdgeAttributes(const void *Node, EdgeIter EI) { + if (EI.isSpecialDep()) + return "color=cyan,style=dashed"; + if (EI.isCtrlDep()) + return "color=blue,style=dashed"; + return ""; + } + + + static std::string getNodeLabel(const SUnit *Node, + const ScheduleDAG *Graph); + static std::string getNodeAttributes(const SUnit *N, + const ScheduleDAG *Graph) { + return "shape=Mrecord"; + } + + static void addCustomGraphFeatures(ScheduleDAG *G, + GraphWriter &GW) { + return G->addCustomGraphFeatures(GW); + } + }; +} + +std::string DOTGraphTraits::getNodeLabel(const SUnit *SU, + const ScheduleDAG *G) { + return G->getGraphNodeLabel(SU); +} + +/// viewGraph - Pop up a ghostview window with the reachable parts of the DAG +/// rendered using 'dot'. +/// +void ScheduleDAG::viewGraph() { +// This code is only for debugging! +#ifndef NDEBUG + ViewGraph(this, "dag." + MF->getFunction()->getName(), + "Scheduling-Units Graph for " + MF->getFunction()->getName() + ':' + + BB->getBasicBlock()->getName()); +#else + cerr << "ScheduleDAG::viewGraph is only available in debug builds on " + << "systems with Graphviz or gv!\n"; +#endif // NDEBUG +} diff --git a/lib/CodeGen/SelectionDAG/CMakeLists.txt b/lib/CodeGen/SelectionDAG/CMakeLists.txt index a3654c24736..186e3d13c90 100644 --- a/lib/CodeGen/SelectionDAG/CMakeLists.txt +++ b/lib/CodeGen/SelectionDAG/CMakeLists.txt @@ -2,15 +2,14 @@ add_llvm_library(LLVMSelectionDAG CallingConvLower.cpp DAGCombiner.cpp FastISel.cpp - LatencyPriorityQueue.cpp LegalizeDAG.cpp LegalizeFloatTypes.cpp LegalizeIntegerTypes.cpp LegalizeTypes.cpp LegalizeTypesGeneric.cpp LegalizeVectorTypes.cpp - ScheduleDAG.cpp - ScheduleDAGEmit.cpp + ScheduleDAGSDNodes.cpp + ScheduleDAGSDNodesEmit.cpp ScheduleDAGFast.cpp ScheduleDAGList.cpp ScheduleDAGRRList.cpp diff --git a/lib/CodeGen/SelectionDAG/LatencyPriorityQueue.cpp b/lib/CodeGen/SelectionDAG/LatencyPriorityQueue.cpp deleted file mode 100644 index ae73f202685..00000000000 --- a/lib/CodeGen/SelectionDAG/LatencyPriorityQueue.cpp +++ /dev/null @@ -1,165 +0,0 @@ -//===---- LatencyPriorityQueue.cpp - A latency-oriented priority queue ----===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements the LatencyPriorityQueue class, which is a -// SchedulingPriorityQueue that schedules using latency information to -// reduce the length of the critical path through the basic block. -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "scheduler" -#include "LatencyPriorityQueue.h" -#include "llvm/Support/Debug.h" -using namespace llvm; - -bool latency_sort::operator()(const SUnit *LHS, const SUnit *RHS) const { - unsigned LHSNum = LHS->NodeNum; - unsigned RHSNum = RHS->NodeNum; - - // The most important heuristic is scheduling the critical path. - unsigned LHSLatency = PQ->getLatency(LHSNum); - unsigned RHSLatency = PQ->getLatency(RHSNum); - if (LHSLatency < RHSLatency) return true; - if (LHSLatency > RHSLatency) return false; - - // After that, if two nodes have identical latencies, look to see if one will - // unblock more other nodes than the other. - unsigned LHSBlocked = PQ->getNumSolelyBlockNodes(LHSNum); - unsigned RHSBlocked = PQ->getNumSolelyBlockNodes(RHSNum); - if (LHSBlocked < RHSBlocked) return true; - if (LHSBlocked > RHSBlocked) return false; - - // Finally, just to provide a stable ordering, use the node number as a - // deciding factor. - return LHSNum < RHSNum; -} - - -/// CalcNodePriority - Calculate the maximal path from the node to the exit. -/// -int LatencyPriorityQueue::CalcLatency(const SUnit &SU) { - int &Latency = Latencies[SU.NodeNum]; - if (Latency != -1) - return Latency; - - std::vector WorkList; - WorkList.push_back(&SU); - while (!WorkList.empty()) { - const SUnit *Cur = WorkList.back(); - bool AllDone = true; - int MaxSuccLatency = 0; - for (SUnit::const_succ_iterator I = Cur->Succs.begin(),E = Cur->Succs.end(); - I != E; ++I) { - int SuccLatency = Latencies[I->Dep->NodeNum]; - if (SuccLatency == -1) { - AllDone = false; - WorkList.push_back(I->Dep); - } else { - MaxSuccLatency = std::max(MaxSuccLatency, SuccLatency); - } - } - if (AllDone) { - Latencies[Cur->NodeNum] = MaxSuccLatency + Cur->Latency; - WorkList.pop_back(); - } - } - - return Latency; -} - -/// CalculatePriorities - Calculate priorities of all scheduling units. -void LatencyPriorityQueue::CalculatePriorities() { - Latencies.assign(SUnits->size(), -1); - NumNodesSolelyBlocking.assign(SUnits->size(), 0); - - // For each node, calculate the maximal path from the node to the exit. - std::vector > WorkList; - for (unsigned i = 0, e = SUnits->size(); i != e; ++i) { - const SUnit *SU = &(*SUnits)[i]; - if (SU->Succs.empty()) - WorkList.push_back(std::make_pair(SU, 0U)); - } - - while (!WorkList.empty()) { - const SUnit *SU = WorkList.back().first; - unsigned SuccLat = WorkList.back().second; - WorkList.pop_back(); - int &Latency = Latencies[SU->NodeNum]; - if (Latency == -1 || (SU->Latency + SuccLat) > (unsigned)Latency) { - Latency = SU->Latency + SuccLat; - for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); - I != E; ++I) - WorkList.push_back(std::make_pair(I->Dep, Latency)); - } - } -} - -/// getSingleUnscheduledPred - If there is exactly one unscheduled predecessor -/// of SU, return it, otherwise return null. -SUnit *LatencyPriorityQueue::getSingleUnscheduledPred(SUnit *SU) { - SUnit *OnlyAvailablePred = 0; - for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); - I != E; ++I) { - SUnit &Pred = *I->Dep; - if (!Pred.isScheduled) { - // We found an available, but not scheduled, predecessor. If it's the - // only one we have found, keep track of it... otherwise give up. - if (OnlyAvailablePred && OnlyAvailablePred != &Pred) - return 0; - OnlyAvailablePred = &Pred; - } - } - - return OnlyAvailablePred; -} - -void LatencyPriorityQueue::push_impl(SUnit *SU) { - // Look at all of the successors of this node. Count the number of nodes that - // this node is the sole unscheduled node for. - unsigned NumNodesBlocking = 0; - for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); - I != E; ++I) - if (getSingleUnscheduledPred(I->Dep) == SU) - ++NumNodesBlocking; - NumNodesSolelyBlocking[SU->NodeNum] = NumNodesBlocking; - - Queue.push(SU); -} - - -// ScheduledNode - As nodes are scheduled, we look to see if there are any -// successor nodes that have a single unscheduled predecessor. If so, that -// single predecessor has a higher priority, since scheduling it will make -// the node available. -void LatencyPriorityQueue::ScheduledNode(SUnit *SU) { - for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); - I != E; ++I) - AdjustPriorityOfUnscheduledPreds(I->Dep); -} - -/// AdjustPriorityOfUnscheduledPreds - One of the predecessors of SU was just -/// scheduled. If SU is not itself available, then there is at least one -/// predecessor node that has not been scheduled yet. If SU has exactly ONE -/// unscheduled predecessor, we want to increase its priority: it getting -/// scheduled will make this node available, so it is better than some other -/// node of the same priority that will not make a node available. -void LatencyPriorityQueue::AdjustPriorityOfUnscheduledPreds(SUnit *SU) { - if (SU->isAvailable) return; // All preds scheduled. - - SUnit *OnlyAvailablePred = getSingleUnscheduledPred(SU); - if (OnlyAvailablePred == 0 || !OnlyAvailablePred->isAvailable) return; - - // Okay, we found a single predecessor that is available, but not scheduled. - // Since it is available, it must be in the priority queue. First remove it. - remove(OnlyAvailablePred); - - // Reinsert the node into the priority queue, which recomputes its - // NumNodesSolelyBlocking value. - push(OnlyAvailablePred); -} diff --git a/lib/CodeGen/SelectionDAG/LatencyPriorityQueue.h b/lib/CodeGen/SelectionDAG/LatencyPriorityQueue.h deleted file mode 100644 index 5b2b02b49d7..00000000000 --- a/lib/CodeGen/SelectionDAG/LatencyPriorityQueue.h +++ /dev/null @@ -1,124 +0,0 @@ -//===---- LatencyPriorityQueue.h - A latency-oriented priority queue ------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file declares the LatencyPriorityQueue class, which is a -// SchedulingPriorityQueue that schedules using latency information to -// reduce the length of the critical path through the basic block. -// -//===----------------------------------------------------------------------===// - -#ifndef LATENCY_PRIORITY_QUEUE_H -#define LATENCY_PRIORITY_QUEUE_H - -#include "llvm/CodeGen/ScheduleDAG.h" -#include "llvm/ADT/PriorityQueue.h" - -namespace llvm { - class LatencyPriorityQueue; - - /// Sorting functions for the Available queue. - struct latency_sort : public std::binary_function { - LatencyPriorityQueue *PQ; - explicit latency_sort(LatencyPriorityQueue *pq) : PQ(pq) {} - - bool operator()(const SUnit* left, const SUnit* right) const; - }; - - class LatencyPriorityQueue : public SchedulingPriorityQueue { - // SUnits - The SUnits for the current graph. - std::vector *SUnits; - - // Latencies - The latency (max of latency from this node to the bb exit) - // for each node. - std::vector Latencies; - - /// NumNodesSolelyBlocking - This vector contains, for every node in the - /// Queue, the number of nodes that the node is the sole unscheduled - /// predecessor for. This is used as a tie-breaker heuristic for better - /// mobility. - std::vector NumNodesSolelyBlocking; - - PriorityQueue, latency_sort> Queue; - public: - LatencyPriorityQueue() : Queue(latency_sort(this)) { - } - - void initNodes(std::vector &sunits) { - SUnits = &sunits; - // Calculate node priorities. - CalculatePriorities(); - } - - void addNode(const SUnit *SU) { - Latencies.resize(SUnits->size(), -1); - NumNodesSolelyBlocking.resize(SUnits->size(), 0); - CalcLatency(*SU); - } - - void updateNode(const SUnit *SU) { - Latencies[SU->NodeNum] = -1; - CalcLatency(*SU); - } - - void releaseState() { - SUnits = 0; - Latencies.clear(); - } - - unsigned getLatency(unsigned NodeNum) const { - assert(NodeNum < Latencies.size()); - return Latencies[NodeNum]; - } - - unsigned getNumSolelyBlockNodes(unsigned NodeNum) const { - assert(NodeNum < NumNodesSolelyBlocking.size()); - return NumNodesSolelyBlocking[NodeNum]; - } - - unsigned size() const { return Queue.size(); } - - bool empty() const { return Queue.empty(); } - - virtual void push(SUnit *U) { - push_impl(U); - } - void push_impl(SUnit *U); - - void push_all(const std::vector &Nodes) { - for (unsigned i = 0, e = Nodes.size(); i != e; ++i) - push_impl(Nodes[i]); - } - - SUnit *pop() { - if (empty()) return NULL; - SUnit *V = Queue.top(); - Queue.pop(); - return V; - } - - void remove(SUnit *SU) { - assert(!Queue.empty() && "Not in queue!"); - Queue.erase_one(SU); - } - - // ScheduledNode - As nodes are scheduled, we look to see if there are any - // successor nodes that have a single unscheduled predecessor. If so, that - // single predecessor has a higher priority, since scheduling it will make - // the node available. - void ScheduledNode(SUnit *Node); - - private: - void CalculatePriorities(); - int CalcLatency(const SUnit &SU); - void AdjustPriorityOfUnscheduledPreds(SUnit *SU); - SUnit *getSingleUnscheduledPred(SUnit *SU); - }; -} - -#endif diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp deleted file mode 100644 index c2e21752385..00000000000 --- a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp +++ /dev/null @@ -1,522 +0,0 @@ -//===---- ScheduleDAG.cpp - Implement the ScheduleDAG class ---------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This implements the ScheduleDAG class, which is a base class used by -// scheduling implementation classes. -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "pre-RA-sched" -#include "llvm/CodeGen/ScheduleDAG.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -using namespace llvm; - -ScheduleDAG::ScheduleDAG(SelectionDAG *dag, MachineBasicBlock *bb, - const TargetMachine &tm) - : DAG(dag), BB(bb), TM(tm), MRI(BB->getParent()->getRegInfo()) { - TII = TM.getInstrInfo(); - MF = BB->getParent(); - TRI = TM.getRegisterInfo(); - TLI = TM.getTargetLowering(); - ConstPool = MF->getConstantPool(); -} - -/// CheckForPhysRegDependency - Check if the dependency between def and use of -/// a specified operand is a physical register dependency. If so, returns the -/// register and the cost of copying the register. -static void CheckForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op, - const TargetRegisterInfo *TRI, - const TargetInstrInfo *TII, - unsigned &PhysReg, int &Cost) { - if (Op != 2 || User->getOpcode() != ISD::CopyToReg) - return; - - unsigned Reg = cast(User->getOperand(1))->getReg(); - if (TargetRegisterInfo::isVirtualRegister(Reg)) - return; - - unsigned ResNo = User->getOperand(2).getResNo(); - if (Def->isMachineOpcode()) { - const TargetInstrDesc &II = TII->get(Def->getMachineOpcode()); - if (ResNo >= II.getNumDefs() && - II.ImplicitDefs[ResNo - II.getNumDefs()] == Reg) { - PhysReg = Reg; - const TargetRegisterClass *RC = - TRI->getPhysicalRegisterRegClass(Reg, Def->getValueType(ResNo)); - Cost = RC->getCopyCost(); - } - } -} - -SUnit *ScheduleDAG::Clone(SUnit *Old) { - SUnit *SU = NewSUnit(Old->getNode()); - SU->OrigNode = Old->OrigNode; - SU->Latency = Old->Latency; - SU->isTwoAddress = Old->isTwoAddress; - SU->isCommutable = Old->isCommutable; - SU->hasPhysRegDefs = Old->hasPhysRegDefs; - return SU; -} - - -/// BuildSchedUnits - Build SUnits from the selection dag that we are input. -/// This SUnit graph is similar to the SelectionDAG, but represents flagged -/// together nodes with a single SUnit. -void ScheduleDAG::BuildSchedUnits() { - // For post-regalloc scheduling, build the SUnits from the MachineInstrs - // in the MachineBasicBlock. - if (!DAG) { - BuildSchedUnitsFromMBB(); - return; - } - - // Reserve entries in the vector for each of the SUnits we are creating. This - // ensure that reallocation of the vector won't happen, so SUnit*'s won't get - // invalidated. - SUnits.reserve(DAG->allnodes_size()); - - // During scheduling, the NodeId field of SDNode is used to map SDNodes - // to their associated SUnits by holding SUnits table indices. A value - // of -1 means the SDNode does not yet have an associated SUnit. - for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(), - E = DAG->allnodes_end(); NI != E; ++NI) - NI->setNodeId(-1); - - for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(), - E = DAG->allnodes_end(); NI != E; ++NI) { - if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate. - continue; - - // If this node has already been processed, stop now. - if (NI->getNodeId() != -1) continue; - - SUnit *NodeSUnit = NewSUnit(NI); - - // See if anything is flagged to this node, if so, add them to flagged - // nodes. Nodes can have at most one flag input and one flag output. Flags - // are required the be the last operand and result of a node. - - // Scan up to find flagged preds. - SDNode *N = NI; - if (N->getNumOperands() && - N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Flag) { - do { - N = N->getOperand(N->getNumOperands()-1).getNode(); - assert(N->getNodeId() == -1 && "Node already inserted!"); - N->setNodeId(NodeSUnit->NodeNum); - } while (N->getNumOperands() && - N->getOperand(N->getNumOperands()-1).getValueType()== MVT::Flag); - } - - // Scan down to find any flagged succs. - N = NI; - while (N->getValueType(N->getNumValues()-1) == MVT::Flag) { - SDValue FlagVal(N, N->getNumValues()-1); - - // There are either zero or one users of the Flag result. - bool HasFlagUse = false; - for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); - UI != E; ++UI) - if (FlagVal.isOperandOf(*UI)) { - HasFlagUse = true; - assert(N->getNodeId() == -1 && "Node already inserted!"); - N->setNodeId(NodeSUnit->NodeNum); - N = *UI; - break; - } - if (!HasFlagUse) break; - } - - // If there are flag operands involved, N is now the bottom-most node - // of the sequence of nodes that are flagged together. - // Update the SUnit. - NodeSUnit->setNode(N); - assert(N->getNodeId() == -1 && "Node already inserted!"); - N->setNodeId(NodeSUnit->NodeNum); - - ComputeLatency(NodeSUnit); - } - - // Pass 2: add the preds, succs, etc. - for (unsigned su = 0, e = SUnits.size(); su != e; ++su) { - SUnit *SU = &SUnits[su]; - SDNode *MainNode = SU->getNode(); - - if (MainNode->isMachineOpcode()) { - unsigned Opc = MainNode->getMachineOpcode(); - const TargetInstrDesc &TID = TII->get(Opc); - for (unsigned i = 0; i != TID.getNumOperands(); ++i) { - if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) { - SU->isTwoAddress = true; - break; - } - } - if (TID.isCommutable()) - SU->isCommutable = true; - } - - // Find all predecessors and successors of the group. - for (SDNode *N = SU->getNode(); N; N = N->getFlaggedNode()) { - if (N->isMachineOpcode() && - TII->get(N->getMachineOpcode()).getImplicitDefs() && - CountResults(N) > TII->get(N->getMachineOpcode()).getNumDefs()) - SU->hasPhysRegDefs = true; - - for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { - SDNode *OpN = N->getOperand(i).getNode(); - if (isPassiveNode(OpN)) continue; // Not scheduled. - SUnit *OpSU = &SUnits[OpN->getNodeId()]; - assert(OpSU && "Node has no SUnit!"); - if (OpSU == SU) continue; // In the same group. - - MVT OpVT = N->getOperand(i).getValueType(); - assert(OpVT != MVT::Flag && "Flagged nodes should be in same sunit!"); - bool isChain = OpVT == MVT::Other; - - unsigned PhysReg = 0; - int Cost = 1; - // Determine if this is a physical register dependency. - CheckForPhysRegDependency(OpN, N, i, TRI, TII, PhysReg, Cost); - SU->addPred(OpSU, isChain, false, PhysReg, Cost); - } - } - } -} - -void ScheduleDAG::BuildSchedUnitsFromMBB() { - SUnits.clear(); - SUnits.reserve(BB->size()); - - std::vector PendingLoads; - SUnit *Terminator = 0; - SUnit *Chain = 0; - SUnit *Defs[TargetRegisterInfo::FirstVirtualRegister] = {}; - std::vector Uses[TargetRegisterInfo::FirstVirtualRegister] = {}; - int Cost = 1; // FIXME - - for (MachineBasicBlock::iterator MII = BB->end(), MIE = BB->begin(); - MII != MIE; --MII) { - MachineInstr *MI = prior(MII); - SUnit *SU = NewSUnit(MI); - - for (unsigned j = 0, n = MI->getNumOperands(); j != n; ++j) { - const MachineOperand &MO = MI->getOperand(j); - if (!MO.isReg()) continue; - unsigned Reg = MO.getReg(); - if (Reg == 0) continue; - - assert(TRI->isPhysicalRegister(Reg) && "Virtual register encountered!"); - std::vector &UseList = Uses[Reg]; - SUnit *&Def = Defs[Reg]; - // Optionally add output and anti dependences - if (Def && Def != SU) - Def->addPred(SU, /*isCtrl=*/true, /*isSpecial=*/false, - /*PhyReg=*/Reg, Cost); - for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) { - SUnit *&Def = Defs[*Alias]; - if (Def && Def != SU) - Def->addPred(SU, /*isCtrl=*/true, /*isSpecial=*/false, - /*PhyReg=*/*Alias, Cost); - } - - if (MO.isDef()) { - // Add any data dependencies. - for (unsigned i = 0, e = UseList.size(); i != e; ++i) - if (UseList[i] != SU) - UseList[i]->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false, - /*PhysReg=*/Reg, Cost); - for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) { - std::vector &UseList = Uses[*Alias]; - for (unsigned i = 0, e = UseList.size(); i != e; ++i) - if (UseList[i] != SU) - UseList[i]->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false, - /*PhysReg=*/*Alias, Cost); - } - - UseList.clear(); - Def = SU; - } else { - UseList.push_back(SU); - } - } - bool False = false; - bool True = true; - if (!MI->isSafeToMove(TII, False)) { - if (Chain) - Chain->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false); - for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) - PendingLoads[k]->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false); - PendingLoads.clear(); - Chain = SU; - } else if (!MI->isSafeToMove(TII, True)) { - if (Chain) - Chain->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false); - PendingLoads.push_back(SU); - } - if (Terminator && SU->Succs.empty()) - Terminator->addPred(SU, /*isCtrl=*/false, /*isSpecial=*/false); - if (MI->getDesc().isTerminator()) - Terminator = SU; - } -} - -void ScheduleDAG::ComputeLatency(SUnit *SU) { - const InstrItineraryData &InstrItins = TM.getInstrItineraryData(); - - // Compute the latency for the node. We use the sum of the latencies for - // all nodes flagged together into this SUnit. - if (InstrItins.isEmpty()) { - // No latency information. - SU->Latency = 1; - return; - } - - SU->Latency = 0; - for (SDNode *N = SU->getNode(); N; N = N->getFlaggedNode()) { - if (N->isMachineOpcode()) { - unsigned SchedClass = TII->get(N->getMachineOpcode()).getSchedClass(); - const InstrStage *S = InstrItins.begin(SchedClass); - const InstrStage *E = InstrItins.end(SchedClass); - for (; S != E; ++S) - SU->Latency += S->Cycles; - } - } -} - -/// CalculateDepths - compute depths using algorithms for the longest -/// paths in the DAG -void ScheduleDAG::CalculateDepths() { - unsigned DAGSize = SUnits.size(); - std::vector WorkList; - WorkList.reserve(DAGSize); - - // Initialize the data structures - for (unsigned i = 0, e = DAGSize; i != e; ++i) { - SUnit *SU = &SUnits[i]; - unsigned Degree = SU->Preds.size(); - // Temporarily use the Depth field as scratch space for the degree count. - SU->Depth = Degree; - - // Is it a node without dependencies? - if (Degree == 0) { - assert(SU->Preds.empty() && "SUnit should have no predecessors"); - // Collect leaf nodes - WorkList.push_back(SU); - } - } - - // Process nodes in the topological order - while (!WorkList.empty()) { - SUnit *SU = WorkList.back(); - WorkList.pop_back(); - unsigned SUDepth = 0; - - // Use dynamic programming: - // When current node is being processed, all of its dependencies - // are already processed. - // So, just iterate over all predecessors and take the longest path - for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); - I != E; ++I) { - unsigned PredDepth = I->Dep->Depth; - if (PredDepth+1 > SUDepth) { - SUDepth = PredDepth + 1; - } - } - - SU->Depth = SUDepth; - - // Update degrees of all nodes depending on current SUnit - for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); - I != E; ++I) { - SUnit *SU = I->Dep; - if (!--SU->Depth) - // If all dependencies of the node are processed already, - // then the longest path for the node can be computed now - WorkList.push_back(SU); - } - } -} - -/// CalculateHeights - compute heights using algorithms for the longest -/// paths in the DAG -void ScheduleDAG::CalculateHeights() { - unsigned DAGSize = SUnits.size(); - std::vector WorkList; - WorkList.reserve(DAGSize); - - // Initialize the data structures - for (unsigned i = 0, e = DAGSize; i != e; ++i) { - SUnit *SU = &SUnits[i]; - unsigned Degree = SU->Succs.size(); - // Temporarily use the Height field as scratch space for the degree count. - SU->Height = Degree; - - // Is it a node without dependencies? - if (Degree == 0) { - assert(SU->Succs.empty() && "Something wrong"); - assert(WorkList.empty() && "Should be empty"); - // Collect leaf nodes - WorkList.push_back(SU); - } - } - - // Process nodes in the topological order - while (!WorkList.empty()) { - SUnit *SU = WorkList.back(); - WorkList.pop_back(); - unsigned SUHeight = 0; - - // Use dynamic programming: - // When current node is being processed, all of its dependencies - // are already processed. - // So, just iterate over all successors and take the longest path - for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); - I != E; ++I) { - unsigned SuccHeight = I->Dep->Height; - if (SuccHeight+1 > SUHeight) { - SUHeight = SuccHeight + 1; - } - } - - SU->Height = SUHeight; - - // Update degrees of all nodes depending on current SUnit - for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); - I != E; ++I) { - SUnit *SU = I->Dep; - if (!--SU->Height) - // If all dependencies of the node are processed already, - // then the longest path for the node can be computed now - WorkList.push_back(SU); - } - } -} - -/// CountResults - The results of target nodes have register or immediate -/// operands first, then an optional chain, and optional flag operands (which do -/// not go into the resulting MachineInstr). -unsigned ScheduleDAG::CountResults(SDNode *Node) { - unsigned N = Node->getNumValues(); - while (N && Node->getValueType(N - 1) == MVT::Flag) - --N; - if (N && Node->getValueType(N - 1) == MVT::Other) - --N; // Skip over chain result. - return N; -} - -/// CountOperands - The inputs to target nodes have any actual inputs first, -/// followed by special operands that describe memory references, then an -/// optional chain operand, then an optional flag operand. Compute the number -/// of actual operands that will go into the resulting MachineInstr. -unsigned ScheduleDAG::CountOperands(SDNode *Node) { - unsigned N = ComputeMemOperandsEnd(Node); - while (N && isa(Node->getOperand(N - 1).getNode())) - --N; // Ignore MEMOPERAND nodes - return N; -} - -/// ComputeMemOperandsEnd - Find the index one past the last MemOperandSDNode -/// operand -unsigned ScheduleDAG::ComputeMemOperandsEnd(SDNode *Node) { - unsigned N = Node->getNumOperands(); - while (N && Node->getOperand(N - 1).getValueType() == MVT::Flag) - --N; - if (N && Node->getOperand(N - 1).getValueType() == MVT::Other) - --N; // Ignore chain if it exists. - return N; -} - - -/// dump - dump the schedule. -void ScheduleDAG::dumpSchedule() const { - for (unsigned i = 0, e = Sequence.size(); i != e; i++) { - if (SUnit *SU = Sequence[i]) - SU->dump(this); - else - cerr << "**** NOOP ****\n"; - } -} - - -/// Run - perform scheduling. -/// -void ScheduleDAG::Run() { - Schedule(); - - DOUT << "*** Final schedule ***\n"; - DEBUG(dumpSchedule()); - DOUT << "\n"; -} - -/// SUnit - Scheduling unit. It's an wrapper around either a single SDNode or -/// a group of nodes flagged together. -void SUnit::print(raw_ostream &O, const ScheduleDAG *G) const { - O << "SU(" << NodeNum << "): "; - if (getNode()) { - SmallVector FlaggedNodes; - for (SDNode *N = getNode(); N; N = N->getFlaggedNode()) - FlaggedNodes.push_back(N); - while (!FlaggedNodes.empty()) { - O << " "; - FlaggedNodes.back()->print(O, G->DAG); - O << "\n"; - FlaggedNodes.pop_back(); - } - } else { - O << "CROSS RC COPY\n"; - } -} - -void SUnit::dump(const ScheduleDAG *G) const { - print(errs(), G); -} - -void SUnit::dumpAll(const ScheduleDAG *G) const { - dump(G); - - cerr << " # preds left : " << NumPredsLeft << "\n"; - cerr << " # succs left : " << NumSuccsLeft << "\n"; - cerr << " Latency : " << Latency << "\n"; - cerr << " Depth : " << Depth << "\n"; - cerr << " Height : " << Height << "\n"; - - if (Preds.size() != 0) { - cerr << " Predecessors:\n"; - for (SUnit::const_succ_iterator I = Preds.begin(), E = Preds.end(); - I != E; ++I) { - if (I->isCtrl) - cerr << " ch #"; - else - cerr << " val #"; - cerr << I->Dep << " - SU(" << I->Dep->NodeNum << ")"; - if (I->isSpecial) - cerr << " *"; - cerr << "\n"; - } - } - if (Succs.size() != 0) { - cerr << " Successors:\n"; - for (SUnit::const_succ_iterator I = Succs.begin(), E = Succs.end(); - I != E; ++I) { - if (I->isCtrl) - cerr << " ch #"; - else - cerr << " val #"; - cerr << I->Dep << " - SU(" << I->Dep->NodeNum << ")"; - if (I->isSpecial) - cerr << " *"; - cerr << "\n"; - } - } - cerr << "\n"; -} diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGEmit.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGEmit.cpp deleted file mode 100644 index 0c679734ea1..00000000000 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGEmit.cpp +++ /dev/null @@ -1,717 +0,0 @@ -//===---- ScheduleDAGEmit.cpp - Emit routines for the ScheduleDAG class ---===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This implements the Emit routines for the ScheduleDAG class, which creates -// MachineInstrs according to the computed schedule. -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "pre-RA-sched" -#include "llvm/CodeGen/ScheduleDAG.h" -#include "llvm/CodeGen/MachineConstantPool.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetData.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/MathExtras.h" -using namespace llvm; - -STATISTIC(NumCommutes, "Number of instructions commuted"); - -/// getInstrOperandRegClass - Return register class of the operand of an -/// instruction of the specified TargetInstrDesc. -static const TargetRegisterClass* -getInstrOperandRegClass(const TargetRegisterInfo *TRI, - const TargetInstrInfo *TII, const TargetInstrDesc &II, - unsigned Op) { - if (Op >= II.getNumOperands()) { - assert(II.isVariadic() && "Invalid operand # of instruction"); - return NULL; - } - if (II.OpInfo[Op].isLookupPtrRegClass()) - return TII->getPointerRegClass(); - return TRI->getRegClass(II.OpInfo[Op].RegClass); -} - -/// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an -/// implicit physical register output. -void ScheduleDAG::EmitCopyFromReg(SDNode *Node, unsigned ResNo, - bool IsClone, unsigned SrcReg, - DenseMap &VRBaseMap) { - unsigned VRBase = 0; - if (TargetRegisterInfo::isVirtualRegister(SrcReg)) { - // Just use the input register directly! - SDValue Op(Node, ResNo); - if (IsClone) - VRBaseMap.erase(Op); - bool isNew = VRBaseMap.insert(std::make_pair(Op, SrcReg)).second; - isNew = isNew; // Silence compiler warning. - assert(isNew && "Node emitted out of order - early"); - return; - } - - // If the node is only used by a CopyToReg and the dest reg is a vreg, use - // the CopyToReg'd destination register instead of creating a new vreg. - bool MatchReg = true; - const TargetRegisterClass *UseRC = NULL; - for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); - UI != E; ++UI) { - SDNode *User = *UI; - bool Match = true; - if (User->getOpcode() == ISD::CopyToReg && - User->getOperand(2).getNode() == Node && - User->getOperand(2).getResNo() == ResNo) { - unsigned DestReg = cast(User->getOperand(1))->getReg(); - if (TargetRegisterInfo::isVirtualRegister(DestReg)) { - VRBase = DestReg; - Match = false; - } else if (DestReg != SrcReg) - Match = false; - } else { - for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { - SDValue Op = User->getOperand(i); - if (Op.getNode() != Node || Op.getResNo() != ResNo) - continue; - MVT VT = Node->getValueType(Op.getResNo()); - if (VT == MVT::Other || VT == MVT::Flag) - continue; - Match = false; - if (User->isMachineOpcode()) { - const TargetInstrDesc &II = TII->get(User->getMachineOpcode()); - const TargetRegisterClass *RC = - getInstrOperandRegClass(TRI,TII,II,i+II.getNumDefs()); - if (!UseRC) - UseRC = RC; - else if (RC) - assert(UseRC == RC && - "Multiple uses expecting different register classes!"); - } - } - } - MatchReg &= Match; - if (VRBase) - break; - } - - MVT VT = Node->getValueType(ResNo); - const TargetRegisterClass *SrcRC = 0, *DstRC = 0; - SrcRC = TRI->getPhysicalRegisterRegClass(SrcReg, VT); - - // Figure out the register class to create for the destreg. - if (VRBase) { - DstRC = MRI.getRegClass(VRBase); - } else if (UseRC) { - assert(UseRC->hasType(VT) && "Incompatible phys register def and uses!"); - DstRC = UseRC; - } else { - DstRC = TLI->getRegClassFor(VT); - } - - // If all uses are reading from the src physical register and copying the - // register is either impossible or very expensive, then don't create a copy. - if (MatchReg && SrcRC->getCopyCost() < 0) { - VRBase = SrcReg; - } else { - // Create the reg, emit the copy. - VRBase = MRI.createVirtualRegister(DstRC); - bool Emitted = - TII->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, DstRC, SrcRC); - Emitted = Emitted; // Silence compiler warning. - assert(Emitted && "Unable to issue a copy instruction!"); - } - - SDValue Op(Node, ResNo); - if (IsClone) - VRBaseMap.erase(Op); - bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; - isNew = isNew; // Silence compiler warning. - assert(isNew && "Node emitted out of order - early"); -} - -/// getDstOfCopyToRegUse - If the only use of the specified result number of -/// node is a CopyToReg, return its destination register. Return 0 otherwise. -unsigned ScheduleDAG::getDstOfOnlyCopyToRegUse(SDNode *Node, - unsigned ResNo) const { - if (!Node->hasOneUse()) - return 0; - - SDNode *User = *Node->use_begin(); - if (User->getOpcode() == ISD::CopyToReg && - User->getOperand(2).getNode() == Node && - User->getOperand(2).getResNo() == ResNo) { - unsigned Reg = cast(User->getOperand(1))->getReg(); - if (TargetRegisterInfo::isVirtualRegister(Reg)) - return Reg; - } - return 0; -} - -void ScheduleDAG::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, - const TargetInstrDesc &II, - DenseMap &VRBaseMap) { - assert(Node->getMachineOpcode() != TargetInstrInfo::IMPLICIT_DEF && - "IMPLICIT_DEF should have been handled as a special case elsewhere!"); - - for (unsigned i = 0; i < II.getNumDefs(); ++i) { - // If the specific node value is only used by a CopyToReg and the dest reg - // is a vreg, use the CopyToReg'd destination register instead of creating - // a new vreg. - unsigned VRBase = 0; - for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); - UI != E; ++UI) { - SDNode *User = *UI; - if (User->getOpcode() == ISD::CopyToReg && - User->getOperand(2).getNode() == Node && - User->getOperand(2).getResNo() == i) { - unsigned Reg = cast(User->getOperand(1))->getReg(); - if (TargetRegisterInfo::isVirtualRegister(Reg)) { - VRBase = Reg; - MI->addOperand(MachineOperand::CreateReg(Reg, true)); - break; - } - } - } - - // Create the result registers for this node and add the result regs to - // the machine instruction. - if (VRBase == 0) { - const TargetRegisterClass *RC = getInstrOperandRegClass(TRI, TII, II, i); - assert(RC && "Isn't a register operand!"); - VRBase = MRI.createVirtualRegister(RC); - MI->addOperand(MachineOperand::CreateReg(VRBase, true)); - } - - SDValue Op(Node, i); - bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; - isNew = isNew; // Silence compiler warning. - assert(isNew && "Node emitted out of order - early"); - } -} - -/// getVR - Return the virtual register corresponding to the specified result -/// of the specified node. -unsigned ScheduleDAG::getVR(SDValue Op, - DenseMap &VRBaseMap) { - if (Op.isMachineOpcode() && - Op.getMachineOpcode() == TargetInstrInfo::IMPLICIT_DEF) { - // Add an IMPLICIT_DEF instruction before every use. - unsigned VReg = getDstOfOnlyCopyToRegUse(Op.getNode(), Op.getResNo()); - // IMPLICIT_DEF can produce any type of result so its TargetInstrDesc - // does not include operand register class info. - if (!VReg) { - const TargetRegisterClass *RC = TLI->getRegClassFor(Op.getValueType()); - VReg = MRI.createVirtualRegister(RC); - } - BuildMI(BB, TII->get(TargetInstrInfo::IMPLICIT_DEF), VReg); - return VReg; - } - - DenseMap::iterator I = VRBaseMap.find(Op); - assert(I != VRBaseMap.end() && "Node emitted out of order - late"); - return I->second; -} - - -/// AddOperand - Add the specified operand to the specified machine instr. II -/// specifies the instruction information for the node, and IIOpNum is the -/// operand number (in the II) that we are adding. IIOpNum and II are used for -/// assertions only. -void ScheduleDAG::AddOperand(MachineInstr *MI, SDValue Op, - unsigned IIOpNum, - const TargetInstrDesc *II, - DenseMap &VRBaseMap) { - if (Op.isMachineOpcode()) { - // Note that this case is redundant with the final else block, but we - // include it because it is the most common and it makes the logic - // simpler here. - assert(Op.getValueType() != MVT::Other && - Op.getValueType() != MVT::Flag && - "Chain and flag operands should occur at end of operand list!"); - // Get/emit the operand. - unsigned VReg = getVR(Op, VRBaseMap); - const TargetInstrDesc &TID = MI->getDesc(); - bool isOptDef = IIOpNum < TID.getNumOperands() && - TID.OpInfo[IIOpNum].isOptionalDef(); - MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef)); - - // Verify that it is right. - assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); -#ifndef NDEBUG - if (II) { - // There may be no register class for this operand if it is a variadic - // argument (RC will be NULL in this case). In this case, we just assume - // the regclass is ok. - const TargetRegisterClass *RC = - getInstrOperandRegClass(TRI, TII, *II, IIOpNum); - assert((RC || II->isVariadic()) && "Expected reg class info!"); - const TargetRegisterClass *VRC = MRI.getRegClass(VReg); - if (RC && VRC != RC) { - cerr << "Register class of operand and regclass of use don't agree!\n"; - cerr << "Operand = " << IIOpNum << "\n"; - cerr << "Op->Val = "; Op.getNode()->dump(DAG); cerr << "\n"; - cerr << "MI = "; MI->print(cerr); - cerr << "VReg = " << VReg << "\n"; - cerr << "VReg RegClass size = " << VRC->getSize() - << ", align = " << VRC->getAlignment() << "\n"; - cerr << "Expected RegClass size = " << RC->getSize() - << ", align = " << RC->getAlignment() << "\n"; - cerr << "Fatal error, aborting.\n"; - abort(); - } - } -#endif - } else if (ConstantSDNode *C = dyn_cast(Op)) { - MI->addOperand(MachineOperand::CreateImm(C->getZExtValue())); - } else if (ConstantFPSDNode *F = dyn_cast(Op)) { - const ConstantFP *CFP = F->getConstantFPValue(); - MI->addOperand(MachineOperand::CreateFPImm(CFP)); - } else if (RegisterSDNode *R = dyn_cast(Op)) { - MI->addOperand(MachineOperand::CreateReg(R->getReg(), false)); - } else if (GlobalAddressSDNode *TGA = dyn_cast(Op)) { - MI->addOperand(MachineOperand::CreateGA(TGA->getGlobal(),TGA->getOffset())); - } else if (BasicBlockSDNode *BB = dyn_cast(Op)) { - MI->addOperand(MachineOperand::CreateMBB(BB->getBasicBlock())); - } else if (FrameIndexSDNode *FI = dyn_cast(Op)) { - MI->addOperand(MachineOperand::CreateFI(FI->getIndex())); - } else if (JumpTableSDNode *JT = dyn_cast(Op)) { - MI->addOperand(MachineOperand::CreateJTI(JT->getIndex())); - } else if (ConstantPoolSDNode *CP = dyn_cast(Op)) { - int Offset = CP->getOffset(); - unsigned Align = CP->getAlignment(); - const Type *Type = CP->getType(); - // MachineConstantPool wants an explicit alignment. - if (Align == 0) { - Align = TM.getTargetData()->getPreferredTypeAlignmentShift(Type); - if (Align == 0) { - // Alignment of vector types. FIXME! - Align = TM.getTargetData()->getABITypeSize(Type); - Align = Log2_64(Align); - } - } - - unsigned Idx; - if (CP->isMachineConstantPoolEntry()) - Idx = ConstPool->getConstantPoolIndex(CP->getMachineCPVal(), Align); - else - Idx = ConstPool->getConstantPoolIndex(CP->getConstVal(), Align); - MI->addOperand(MachineOperand::CreateCPI(Idx, Offset)); - } else if (ExternalSymbolSDNode *ES = dyn_cast(Op)) { - MI->addOperand(MachineOperand::CreateES(ES->getSymbol())); - } else { - assert(Op.getValueType() != MVT::Other && - Op.getValueType() != MVT::Flag && - "Chain and flag operands should occur at end of operand list!"); - unsigned VReg = getVR(Op, VRBaseMap); - MI->addOperand(MachineOperand::CreateReg(VReg, false)); - - // Verify that it is right. Note that the reg class of the physreg and the - // vreg don't necessarily need to match, but the target copy insertion has - // to be able to handle it. This handles things like copies from ST(0) to - // an FP vreg on x86. - assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); - if (II && !II->isVariadic()) { - assert(getInstrOperandRegClass(TRI, TII, *II, IIOpNum) && - "Don't have operand info for this instruction!"); - } - } -} - -void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO) { - MI->addMemOperand(*MF, MO); -} - -/// getSubRegisterRegClass - Returns the register class of specified register -/// class' "SubIdx"'th sub-register class. -static const TargetRegisterClass* -getSubRegisterRegClass(const TargetRegisterClass *TRC, unsigned SubIdx) { - // Pick the register class of the subregister - TargetRegisterInfo::regclass_iterator I = - TRC->subregclasses_begin() + SubIdx-1; - assert(I < TRC->subregclasses_end() && - "Invalid subregister index for register class"); - return *I; -} - -/// getSuperRegisterRegClass - Returns the register class of a superreg A whose -/// "SubIdx"'th sub-register class is the specified register class and whose -/// type matches the specified type. -static const TargetRegisterClass* -getSuperRegisterRegClass(const TargetRegisterClass *TRC, - unsigned SubIdx, MVT VT) { - // Pick the register class of the superegister for this type - for (TargetRegisterInfo::regclass_iterator I = TRC->superregclasses_begin(), - E = TRC->superregclasses_end(); I != E; ++I) - if ((*I)->hasType(VT) && getSubRegisterRegClass(*I, SubIdx) == TRC) - return *I; - assert(false && "Couldn't find the register class"); - return 0; -} - -/// EmitSubregNode - Generate machine code for subreg nodes. -/// -void ScheduleDAG::EmitSubregNode(SDNode *Node, - DenseMap &VRBaseMap) { - unsigned VRBase = 0; - unsigned Opc = Node->getMachineOpcode(); - - // If the node is only used by a CopyToReg and the dest reg is a vreg, use - // the CopyToReg'd destination register instead of creating a new vreg. - for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); - UI != E; ++UI) { - SDNode *User = *UI; - if (User->getOpcode() == ISD::CopyToReg && - User->getOperand(2).getNode() == Node) { - unsigned DestReg = cast(User->getOperand(1))->getReg(); - if (TargetRegisterInfo::isVirtualRegister(DestReg)) { - VRBase = DestReg; - break; - } - } - } - - if (Opc == TargetInstrInfo::EXTRACT_SUBREG) { - unsigned SubIdx = cast(Node->getOperand(1))->getZExtValue(); - - // Create the extract_subreg machine instruction. - MachineInstr *MI = BuildMI(*MF, TII->get(TargetInstrInfo::EXTRACT_SUBREG)); - - // Figure out the register class to create for the destreg. - unsigned VReg = getVR(Node->getOperand(0), VRBaseMap); - const TargetRegisterClass *TRC = MRI.getRegClass(VReg); - const TargetRegisterClass *SRC = getSubRegisterRegClass(TRC, SubIdx); - - if (VRBase) { - // Grab the destination register -#ifndef NDEBUG - const TargetRegisterClass *DRC = MRI.getRegClass(VRBase); - assert(SRC && DRC && SRC == DRC && - "Source subregister and destination must have the same class"); -#endif - } else { - // Create the reg - assert(SRC && "Couldn't find source register class"); - VRBase = MRI.createVirtualRegister(SRC); - } - - // Add def, source, and subreg index - MI->addOperand(MachineOperand::CreateReg(VRBase, true)); - AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap); - MI->addOperand(MachineOperand::CreateImm(SubIdx)); - BB->push_back(MI); - } else if (Opc == TargetInstrInfo::INSERT_SUBREG || - Opc == TargetInstrInfo::SUBREG_TO_REG) { - SDValue N0 = Node->getOperand(0); - SDValue N1 = Node->getOperand(1); - SDValue N2 = Node->getOperand(2); - unsigned SubReg = getVR(N1, VRBaseMap); - unsigned SubIdx = cast(N2)->getZExtValue(); - - - // Figure out the register class to create for the destreg. - const TargetRegisterClass *TRC = 0; - if (VRBase) { - TRC = MRI.getRegClass(VRBase); - } else { - TRC = getSuperRegisterRegClass(MRI.getRegClass(SubReg), SubIdx, - Node->getValueType(0)); - assert(TRC && "Couldn't determine register class for insert_subreg"); - VRBase = MRI.createVirtualRegister(TRC); // Create the reg - } - - // Create the insert_subreg or subreg_to_reg machine instruction. - MachineInstr *MI = BuildMI(*MF, TII->get(Opc)); - MI->addOperand(MachineOperand::CreateReg(VRBase, true)); - - // If creating a subreg_to_reg, then the first input operand - // is an implicit value immediate, otherwise it's a register - if (Opc == TargetInstrInfo::SUBREG_TO_REG) { - const ConstantSDNode *SD = cast(N0); - MI->addOperand(MachineOperand::CreateImm(SD->getZExtValue())); - } else - AddOperand(MI, N0, 0, 0, VRBaseMap); - // Add the subregster being inserted - AddOperand(MI, N1, 0, 0, VRBaseMap); - MI->addOperand(MachineOperand::CreateImm(SubIdx)); - BB->push_back(MI); - } else - assert(0 && "Node is not insert_subreg, extract_subreg, or subreg_to_reg"); - - SDValue Op(Node, 0); - bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; - isNew = isNew; // Silence compiler warning. - assert(isNew && "Node emitted out of order - early"); -} - -/// EmitNode - Generate machine code for an node and needed dependencies. -/// -void ScheduleDAG::EmitNode(SDNode *Node, bool IsClone, - DenseMap &VRBaseMap) { - // If machine instruction - if (Node->isMachineOpcode()) { - unsigned Opc = Node->getMachineOpcode(); - - // Handle subreg insert/extract specially - if (Opc == TargetInstrInfo::EXTRACT_SUBREG || - Opc == TargetInstrInfo::INSERT_SUBREG || - Opc == TargetInstrInfo::SUBREG_TO_REG) { - EmitSubregNode(Node, VRBaseMap); - return; - } - - if (Opc == TargetInstrInfo::IMPLICIT_DEF) - // We want a unique VR for each IMPLICIT_DEF use. - return; - - const TargetInstrDesc &II = TII->get(Opc); - unsigned NumResults = CountResults(Node); - unsigned NodeOperands = CountOperands(Node); - unsigned MemOperandsEnd = ComputeMemOperandsEnd(Node); - bool HasPhysRegOuts = (NumResults > II.getNumDefs()) && - II.getImplicitDefs() != 0; -#ifndef NDEBUG - unsigned NumMIOperands = NodeOperands + NumResults; - assert((II.getNumOperands() == NumMIOperands || - HasPhysRegOuts || II.isVariadic()) && - "#operands for dag node doesn't match .td file!"); -#endif - - // Create the new machine instruction. - MachineInstr *MI = BuildMI(*MF, II); - - // Add result register values for things that are defined by this - // instruction. - if (NumResults) - CreateVirtualRegisters(Node, MI, II, VRBaseMap); - - // Emit all of the actual operands of this instruction, adding them to the - // instruction as appropriate. - for (unsigned i = 0; i != NodeOperands; ++i) - AddOperand(MI, Node->getOperand(i), i+II.getNumDefs(), &II, VRBaseMap); - - // Emit all of the memory operands of this instruction - for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i) - AddMemOperand(MI, cast(Node->getOperand(i))->MO); - - // Commute node if it has been determined to be profitable. - if (CommuteSet.count(Node)) { - MachineInstr *NewMI = TII->commuteInstruction(MI); - if (NewMI == 0) - DOUT << "Sched: COMMUTING FAILED!\n"; - else { - DOUT << "Sched: COMMUTED TO: " << *NewMI; - if (MI != NewMI) { - MF->DeleteMachineInstr(MI); - MI = NewMI; - } - ++NumCommutes; - } - } - - if (II.usesCustomDAGSchedInsertionHook()) - // Insert this instruction into the basic block using a target - // specific inserter which may returns a new basic block. - BB = TLI->EmitInstrWithCustomInserter(MI, BB); - else - BB->push_back(MI); - - // Additional results must be an physical register def. - if (HasPhysRegOuts) { - for (unsigned i = II.getNumDefs(); i < NumResults; ++i) { - unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()]; - if (Node->hasAnyUseOfValue(i)) - EmitCopyFromReg(Node, i, IsClone, Reg, VRBaseMap); - } - } - return; - } - - switch (Node->getOpcode()) { - default: -#ifndef NDEBUG - Node->dump(DAG); -#endif - assert(0 && "This target-independent node should have been selected!"); - break; - case ISD::EntryToken: - assert(0 && "EntryToken should have been excluded from the schedule!"); - break; - case ISD::TokenFactor: // fall thru - break; - case ISD::CopyToReg: { - unsigned SrcReg; - SDValue SrcVal = Node->getOperand(2); - if (RegisterSDNode *R = dyn_cast(SrcVal)) - SrcReg = R->getReg(); - else - SrcReg = getVR(SrcVal, VRBaseMap); - - unsigned DestReg = cast(Node->getOperand(1))->getReg(); - if (SrcReg == DestReg) // Coalesced away the copy? Ignore. - break; - - const TargetRegisterClass *SrcTRC = 0, *DstTRC = 0; - // Get the register classes of the src/dst. - if (TargetRegisterInfo::isVirtualRegister(SrcReg)) - SrcTRC = MRI.getRegClass(SrcReg); - else - SrcTRC = TRI->getPhysicalRegisterRegClass(SrcReg,SrcVal.getValueType()); - - if (TargetRegisterInfo::isVirtualRegister(DestReg)) - DstTRC = MRI.getRegClass(DestReg); - else - DstTRC = TRI->getPhysicalRegisterRegClass(DestReg, - Node->getOperand(1).getValueType()); - TII->copyRegToReg(*BB, BB->end(), DestReg, SrcReg, DstTRC, SrcTRC); - break; - } - case ISD::CopyFromReg: { - unsigned SrcReg = cast(Node->getOperand(1))->getReg(); - EmitCopyFromReg(Node, 0, IsClone, SrcReg, VRBaseMap); - break; - } - case ISD::INLINEASM: { - unsigned NumOps = Node->getNumOperands(); - if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag) - --NumOps; // Ignore the flag operand. - - // Create the inline asm machine instruction. - MachineInstr *MI = BuildMI(*MF, TII->get(TargetInstrInfo::INLINEASM)); - - // Add the asm string as an external symbol operand. - const char *AsmStr = - cast(Node->getOperand(1))->getSymbol(); - MI->addOperand(MachineOperand::CreateES(AsmStr)); - - // Add all of the operand registers to the instruction. - for (unsigned i = 2; i != NumOps;) { - unsigned Flags = - cast(Node->getOperand(i))->getZExtValue(); - unsigned NumVals = Flags >> 3; - - MI->addOperand(MachineOperand::CreateImm(Flags)); - ++i; // Skip the ID value. - - switch (Flags & 7) { - default: assert(0 && "Bad flags!"); - case 2: // Def of register. - for (; NumVals; --NumVals, ++i) { - unsigned Reg = cast(Node->getOperand(i))->getReg(); - MI->addOperand(MachineOperand::CreateReg(Reg, true)); - } - break; - case 6: // Def of earlyclobber register. - for (; NumVals; --NumVals, ++i) { - unsigned Reg = cast(Node->getOperand(i))->getReg(); - MI->addOperand(MachineOperand::CreateReg(Reg, true, false, false, - false, 0, true)); - } - break; - case 1: // Use of register. - case 3: // Immediate. - case 4: // Addressing mode. - // The addressing mode has been selected, just add all of the - // operands to the machine instruction. - for (; NumVals; --NumVals, ++i) - AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap); - break; - } - } - BB->push_back(MI); - break; - } - } -} - -void ScheduleDAG::EmitNoop() { - TII->insertNoop(*BB, BB->end()); -} - -void ScheduleDAG::EmitCrossRCCopy(SUnit *SU, - DenseMap &VRBaseMap) { - for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); - I != E; ++I) { - if (I->isCtrl) continue; // ignore chain preds - if (!I->Dep->getNode()) { - // Copy to physical register. - DenseMap::iterator VRI = VRBaseMap.find(I->Dep); - assert(VRI != VRBaseMap.end() && "Node emitted out of order - late"); - // Find the destination physical register. - unsigned Reg = 0; - for (SUnit::const_succ_iterator II = SU->Succs.begin(), - EE = SU->Succs.end(); II != EE; ++II) { - if (I->Reg) { - Reg = I->Reg; - break; - } - } - assert(I->Reg && "Unknown physical register!"); - TII->copyRegToReg(*BB, BB->end(), Reg, VRI->second, - SU->CopyDstRC, SU->CopySrcRC); - } else { - // Copy from physical register. - assert(I->Reg && "Unknown physical register!"); - unsigned VRBase = MRI.createVirtualRegister(SU->CopyDstRC); - bool isNew = VRBaseMap.insert(std::make_pair(SU, VRBase)).second; - isNew = isNew; // Silence compiler warning. - assert(isNew && "Node emitted out of order - early"); - TII->copyRegToReg(*BB, BB->end(), VRBase, I->Reg, - SU->CopyDstRC, SU->CopySrcRC); - } - break; - } -} - -/// EmitSchedule - Emit the machine code in scheduled order. -MachineBasicBlock *ScheduleDAG::EmitSchedule() { - // For post-regalloc scheduling, we're rescheduling the instructions in the - // block, so start by removing them from the block. - if (!DAG) - while (!BB->empty()) - BB->remove(BB->begin()); - - DenseMap VRBaseMap; - DenseMap CopyVRBaseMap; - for (unsigned i = 0, e = Sequence.size(); i != e; i++) { - SUnit *SU = Sequence[i]; - if (!SU) { - // Null SUnit* is a noop. - EmitNoop(); - continue; - } - - // For post-regalloc scheduling, we already have the instruction; - // just append it to the block. - if (!DAG) { - BB->push_back(SU->getInstr()); - continue; - } - - // For pre-regalloc scheduling, create instructions corresponding to the - // SDNode and any flagged SDNodes and append them to the block. - SmallVector FlaggedNodes; - for (SDNode *N = SU->getNode()->getFlaggedNode(); N; N = N->getFlaggedNode()) - FlaggedNodes.push_back(N); - while (!FlaggedNodes.empty()) { - EmitNode(FlaggedNodes.back(), SU->OrigNode != SU, VRBaseMap); - FlaggedNodes.pop_back(); - } - if (!SU->getNode()) - EmitCrossRCCopy(SU, CopyVRBaseMap); - else - EmitNode(SU->getNode(), SU->OrigNode != SU, VRBaseMap); - } - - return BB; -} diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp index 0af03580f66..8f3198e365b 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "pre-RA-sched" -#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetData.h" @@ -58,7 +58,7 @@ namespace { //===----------------------------------------------------------------------===// /// ScheduleDAGFast - The actual "fast" list scheduler implementation. /// -class VISIBILITY_HIDDEN ScheduleDAGFast : public ScheduleDAG { +class VISIBILITY_HIDDEN ScheduleDAGFast : public ScheduleDAGSDNodes { private: /// AvailableQueue - The priority queue to use for the available SUnits. FastPriorityQueue AvailableQueue; @@ -73,7 +73,7 @@ private: public: ScheduleDAGFast(SelectionDAG *dag, MachineBasicBlock *bb, const TargetMachine &tm) - : ScheduleDAG(dag, bb, tm) {} + : ScheduleDAGSDNodes(dag, bb, tm) {} void Schedule(); diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp index c2c91108a54..e1a24a1a729 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp @@ -19,7 +19,8 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "pre-RA-sched" -#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/LatencyPriorityQueue.h" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAGISel.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -30,7 +31,6 @@ #include "llvm/Support/Compiler.h" #include "llvm/ADT/PriorityQueue.h" #include "llvm/ADT/Statistic.h" -#include "LatencyPriorityQueue.h" #include using namespace llvm; @@ -46,7 +46,7 @@ namespace { /// ScheduleDAGList - The actual list scheduler implementation. This supports /// top-down scheduling. /// -class VISIBILITY_HIDDEN ScheduleDAGList : public ScheduleDAG { +class VISIBILITY_HIDDEN ScheduleDAGList : public ScheduleDAGSDNodes { private: /// AvailableQueue - The priority queue to use for the available SUnits. /// @@ -66,7 +66,7 @@ public: const TargetMachine &tm, SchedulingPriorityQueue *availqueue, HazardRecognizer *HR) - : ScheduleDAG(dag, bb, tm), + : ScheduleDAGSDNodes(dag, bb, tm), AvailableQueue(availqueue), HazardRec(HR) { } @@ -212,13 +212,13 @@ void ScheduleDAGList::ListScheduleTopDown() { if (!N) break; FoundNode = N; } - + HazardRecognizer::HazardType HT = HazardRec->getHazardType(FoundNode); if (HT == HazardRecognizer::NoHazard) { FoundSUnit = CurSUnit; break; } - + // Remember if this is a noop hazard. HasNoopHazards |= HT == HazardRecognizer::NoopHazard; diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index 7c3f6bbeb2e..45a1ca05444 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -16,7 +16,7 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "pre-RA-sched" -#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetData.h" @@ -53,7 +53,7 @@ namespace { /// ScheduleDAGRRList - The actual register reduction list scheduler /// implementation. This supports both top-down and bottom-up scheduling. /// -class VISIBILITY_HIDDEN ScheduleDAGRRList : public ScheduleDAG { +class VISIBILITY_HIDDEN ScheduleDAGRRList : public ScheduleDAGSDNodes { private: /// isBottomUp - This is true if the scheduling problem is bottom-up, false if /// it is top-down. @@ -77,7 +77,7 @@ public: ScheduleDAGRRList(SelectionDAG *dag, MachineBasicBlock *bb, const TargetMachine &tm, bool isbottomup, bool f, SchedulingPriorityQueue *availqueue) - : ScheduleDAG(dag, bb, tm), isBottomUp(isbottomup), Fast(f), + : ScheduleDAGSDNodes(dag, bb, tm), isBottomUp(isbottomup), Fast(f), AvailableQueue(availqueue) { } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp new file mode 100644 index 00000000000..9d32d9afac5 --- /dev/null +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -0,0 +1,257 @@ +//===--- ScheduleDAGSDNodes.cpp - Implement the ScheduleDAGSDNodes class --===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements the ScheduleDAG class, which is a base class used by +// scheduling implementation classes. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "pre-RA-sched" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" +#include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +using namespace llvm; + +ScheduleDAGSDNodes::ScheduleDAGSDNodes(SelectionDAG *dag, MachineBasicBlock *bb, + const TargetMachine &tm) + : ScheduleDAG(dag, bb, tm) { +} + +SUnit *ScheduleDAGSDNodes::Clone(SUnit *Old) { + SUnit *SU = NewSUnit(Old->getNode()); + SU->OrigNode = Old->OrigNode; + SU->Latency = Old->Latency; + SU->isTwoAddress = Old->isTwoAddress; + SU->isCommutable = Old->isCommutable; + SU->hasPhysRegDefs = Old->hasPhysRegDefs; + return SU; +} + +/// CheckForPhysRegDependency - Check if the dependency between def and use of +/// a specified operand is a physical register dependency. If so, returns the +/// register and the cost of copying the register. +static void CheckForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op, + const TargetRegisterInfo *TRI, + const TargetInstrInfo *TII, + unsigned &PhysReg, int &Cost) { + if (Op != 2 || User->getOpcode() != ISD::CopyToReg) + return; + + unsigned Reg = cast(User->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return; + + unsigned ResNo = User->getOperand(2).getResNo(); + if (Def->isMachineOpcode()) { + const TargetInstrDesc &II = TII->get(Def->getMachineOpcode()); + if (ResNo >= II.getNumDefs() && + II.ImplicitDefs[ResNo - II.getNumDefs()] == Reg) { + PhysReg = Reg; + const TargetRegisterClass *RC = + TRI->getPhysicalRegisterRegClass(Reg, Def->getValueType(ResNo)); + Cost = RC->getCopyCost(); + } + } +} + +/// BuildSchedUnits - Build SUnits from the selection dag that we are input. +/// This SUnit graph is similar to the SelectionDAG, but represents flagged +/// together nodes with a single SUnit. +void ScheduleDAGSDNodes::BuildSchedUnits() { + // Reserve entries in the vector for each of the SUnits we are creating. This + // ensure that reallocation of the vector won't happen, so SUnit*'s won't get + // invalidated. + SUnits.reserve(DAG->allnodes_size()); + + // During scheduling, the NodeId field of SDNode is used to map SDNodes + // to their associated SUnits by holding SUnits table indices. A value + // of -1 means the SDNode does not yet have an associated SUnit. + for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(), + E = DAG->allnodes_end(); NI != E; ++NI) + NI->setNodeId(-1); + + for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(), + E = DAG->allnodes_end(); NI != E; ++NI) { + if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate. + continue; + + // If this node has already been processed, stop now. + if (NI->getNodeId() != -1) continue; + + SUnit *NodeSUnit = NewSUnit(NI); + + // See if anything is flagged to this node, if so, add them to flagged + // nodes. Nodes can have at most one flag input and one flag output. Flags + // are required the be the last operand and result of a node. + + // Scan up to find flagged preds. + SDNode *N = NI; + if (N->getNumOperands() && + N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Flag) { + do { + N = N->getOperand(N->getNumOperands()-1).getNode(); + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); + } while (N->getNumOperands() && + N->getOperand(N->getNumOperands()-1).getValueType()== MVT::Flag); + } + + // Scan down to find any flagged succs. + N = NI; + while (N->getValueType(N->getNumValues()-1) == MVT::Flag) { + SDValue FlagVal(N, N->getNumValues()-1); + + // There are either zero or one users of the Flag result. + bool HasFlagUse = false; + for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); + UI != E; ++UI) + if (FlagVal.isOperandOf(*UI)) { + HasFlagUse = true; + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); + N = *UI; + break; + } + if (!HasFlagUse) break; + } + + // If there are flag operands involved, N is now the bottom-most node + // of the sequence of nodes that are flagged together. + // Update the SUnit. + NodeSUnit->setNode(N); + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); + + ComputeLatency(NodeSUnit); + } + + // Pass 2: add the preds, succs, etc. + for (unsigned su = 0, e = SUnits.size(); su != e; ++su) { + SUnit *SU = &SUnits[su]; + SDNode *MainNode = SU->getNode(); + + if (MainNode->isMachineOpcode()) { + unsigned Opc = MainNode->getMachineOpcode(); + const TargetInstrDesc &TID = TII->get(Opc); + for (unsigned i = 0; i != TID.getNumOperands(); ++i) { + if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) { + SU->isTwoAddress = true; + break; + } + } + if (TID.isCommutable()) + SU->isCommutable = true; + } + + // Find all predecessors and successors of the group. + for (SDNode *N = SU->getNode(); N; N = N->getFlaggedNode()) { + if (N->isMachineOpcode() && + TII->get(N->getMachineOpcode()).getImplicitDefs() && + CountResults(N) > TII->get(N->getMachineOpcode()).getNumDefs()) + SU->hasPhysRegDefs = true; + + for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { + SDNode *OpN = N->getOperand(i).getNode(); + if (isPassiveNode(OpN)) continue; // Not scheduled. + SUnit *OpSU = &SUnits[OpN->getNodeId()]; + assert(OpSU && "Node has no SUnit!"); + if (OpSU == SU) continue; // In the same group. + + MVT OpVT = N->getOperand(i).getValueType(); + assert(OpVT != MVT::Flag && "Flagged nodes should be in same sunit!"); + bool isChain = OpVT == MVT::Other; + + unsigned PhysReg = 0; + int Cost = 1; + // Determine if this is a physical register dependency. + CheckForPhysRegDependency(OpN, N, i, TRI, TII, PhysReg, Cost); + SU->addPred(OpSU, isChain, false, PhysReg, Cost); + } + } + } +} + +void ScheduleDAGSDNodes::ComputeLatency(SUnit *SU) { + const InstrItineraryData &InstrItins = TM.getInstrItineraryData(); + + // Compute the latency for the node. We use the sum of the latencies for + // all nodes flagged together into this SUnit. + if (InstrItins.isEmpty()) { + // No latency information. + SU->Latency = 1; + return; + } + + SU->Latency = 0; + for (SDNode *N = SU->getNode(); N; N = N->getFlaggedNode()) { + if (N->isMachineOpcode()) { + unsigned SchedClass = TII->get(N->getMachineOpcode()).getSchedClass(); + const InstrStage *S = InstrItins.begin(SchedClass); + const InstrStage *E = InstrItins.end(SchedClass); + for (; S != E; ++S) + SU->Latency += S->Cycles; + } + } +} + +/// CountResults - The results of target nodes have register or immediate +/// operands first, then an optional chain, and optional flag operands (which do +/// not go into the resulting MachineInstr). +unsigned ScheduleDAGSDNodes::CountResults(SDNode *Node) { + unsigned N = Node->getNumValues(); + while (N && Node->getValueType(N - 1) == MVT::Flag) + --N; + if (N && Node->getValueType(N - 1) == MVT::Other) + --N; // Skip over chain result. + return N; +} + +/// CountOperands - The inputs to target nodes have any actual inputs first, +/// followed by special operands that describe memory references, then an +/// optional chain operand, then an optional flag operand. Compute the number +/// of actual operands that will go into the resulting MachineInstr. +unsigned ScheduleDAGSDNodes::CountOperands(SDNode *Node) { + unsigned N = ComputeMemOperandsEnd(Node); + while (N && isa(Node->getOperand(N - 1).getNode())) + --N; // Ignore MEMOPERAND nodes + return N; +} + +/// ComputeMemOperandsEnd - Find the index one past the last MemOperandSDNode +/// operand +unsigned ScheduleDAGSDNodes::ComputeMemOperandsEnd(SDNode *Node) { + unsigned N = Node->getNumOperands(); + while (N && Node->getOperand(N - 1).getValueType() == MVT::Flag) + --N; + if (N && Node->getOperand(N - 1).getValueType() == MVT::Other) + --N; // Ignore chain if it exists. + return N; +} + + +void ScheduleDAGSDNodes::dumpNode(const SUnit *SU) const { + if (SU->getNode()) + SU->getNode()->dump(DAG); + else + cerr << "CROSS RC COPY "; + cerr << "\n"; + SmallVector FlaggedNodes; + for (SDNode *N = SU->getNode()->getFlaggedNode(); N; N = N->getFlaggedNode()) + FlaggedNodes.push_back(N); + while (!FlaggedNodes.empty()) { + cerr << " "; + FlaggedNodes.back()->dump(DAG); + cerr << "\n"; + FlaggedNodes.pop_back(); + } +} diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp new file mode 100644 index 00000000000..dc9313bb0c1 --- /dev/null +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp @@ -0,0 +1,661 @@ +//===---- ScheduleDAGEmit.cpp - Emit routines for the ScheduleDAG class ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements the Emit routines for the ScheduleDAG class, which creates +// MachineInstrs according to the computed schedule. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "pre-RA-sched" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" +#include "llvm/CodeGen/MachineConstantPool.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/Target/TargetData.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" +using namespace llvm; + +STATISTIC(NumCommutes, "Number of instructions commuted"); + +/// getInstrOperandRegClass - Return register class of the operand of an +/// instruction of the specified TargetInstrDesc. +static const TargetRegisterClass* +getInstrOperandRegClass(const TargetRegisterInfo *TRI, + const TargetInstrInfo *TII, const TargetInstrDesc &II, + unsigned Op) { + if (Op >= II.getNumOperands()) { + assert(II.isVariadic() && "Invalid operand # of instruction"); + return NULL; + } + if (II.OpInfo[Op].isLookupPtrRegClass()) + return TII->getPointerRegClass(); + return TRI->getRegClass(II.OpInfo[Op].RegClass); +} + +/// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an +/// implicit physical register output. +void ScheduleDAGSDNodes::EmitCopyFromReg(SDNode *Node, unsigned ResNo, + bool IsClone, unsigned SrcReg, + DenseMap &VRBaseMap) { + unsigned VRBase = 0; + if (TargetRegisterInfo::isVirtualRegister(SrcReg)) { + // Just use the input register directly! + SDValue Op(Node, ResNo); + if (IsClone) + VRBaseMap.erase(Op); + bool isNew = VRBaseMap.insert(std::make_pair(Op, SrcReg)).second; + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); + return; + } + + // If the node is only used by a CopyToReg and the dest reg is a vreg, use + // the CopyToReg'd destination register instead of creating a new vreg. + bool MatchReg = true; + const TargetRegisterClass *UseRC = NULL; + for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); + UI != E; ++UI) { + SDNode *User = *UI; + bool Match = true; + if (User->getOpcode() == ISD::CopyToReg && + User->getOperand(2).getNode() == Node && + User->getOperand(2).getResNo() == ResNo) { + unsigned DestReg = cast(User->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(DestReg)) { + VRBase = DestReg; + Match = false; + } else if (DestReg != SrcReg) + Match = false; + } else { + for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { + SDValue Op = User->getOperand(i); + if (Op.getNode() != Node || Op.getResNo() != ResNo) + continue; + MVT VT = Node->getValueType(Op.getResNo()); + if (VT == MVT::Other || VT == MVT::Flag) + continue; + Match = false; + if (User->isMachineOpcode()) { + const TargetInstrDesc &II = TII->get(User->getMachineOpcode()); + const TargetRegisterClass *RC = + getInstrOperandRegClass(TRI,TII,II,i+II.getNumDefs()); + if (!UseRC) + UseRC = RC; + else if (RC) + assert(UseRC == RC && + "Multiple uses expecting different register classes!"); + } + } + } + MatchReg &= Match; + if (VRBase) + break; + } + + MVT VT = Node->getValueType(ResNo); + const TargetRegisterClass *SrcRC = 0, *DstRC = 0; + SrcRC = TRI->getPhysicalRegisterRegClass(SrcReg, VT); + + // Figure out the register class to create for the destreg. + if (VRBase) { + DstRC = MRI.getRegClass(VRBase); + } else if (UseRC) { + assert(UseRC->hasType(VT) && "Incompatible phys register def and uses!"); + DstRC = UseRC; + } else { + DstRC = TLI->getRegClassFor(VT); + } + + // If all uses are reading from the src physical register and copying the + // register is either impossible or very expensive, then don't create a copy. + if (MatchReg && SrcRC->getCopyCost() < 0) { + VRBase = SrcReg; + } else { + // Create the reg, emit the copy. + VRBase = MRI.createVirtualRegister(DstRC); + bool Emitted = + TII->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, DstRC, SrcRC); + Emitted = Emitted; // Silence compiler warning. + assert(Emitted && "Unable to issue a copy instruction!"); + } + + SDValue Op(Node, ResNo); + if (IsClone) + VRBaseMap.erase(Op); + bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); +} + +/// getDstOfCopyToRegUse - If the only use of the specified result number of +/// node is a CopyToReg, return its destination register. Return 0 otherwise. +unsigned ScheduleDAGSDNodes::getDstOfOnlyCopyToRegUse(SDNode *Node, + unsigned ResNo) const { + if (!Node->hasOneUse()) + return 0; + + SDNode *User = *Node->use_begin(); + if (User->getOpcode() == ISD::CopyToReg && + User->getOperand(2).getNode() == Node && + User->getOperand(2).getResNo() == ResNo) { + unsigned Reg = cast(User->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return Reg; + } + return 0; +} + +void ScheduleDAGSDNodes::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, + const TargetInstrDesc &II, + DenseMap &VRBaseMap) { + assert(Node->getMachineOpcode() != TargetInstrInfo::IMPLICIT_DEF && + "IMPLICIT_DEF should have been handled as a special case elsewhere!"); + + for (unsigned i = 0; i < II.getNumDefs(); ++i) { + // If the specific node value is only used by a CopyToReg and the dest reg + // is a vreg, use the CopyToReg'd destination register instead of creating + // a new vreg. + unsigned VRBase = 0; + for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); + UI != E; ++UI) { + SDNode *User = *UI; + if (User->getOpcode() == ISD::CopyToReg && + User->getOperand(2).getNode() == Node && + User->getOperand(2).getResNo() == i) { + unsigned Reg = cast(User->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + VRBase = Reg; + MI->addOperand(MachineOperand::CreateReg(Reg, true)); + break; + } + } + } + + // Create the result registers for this node and add the result regs to + // the machine instruction. + if (VRBase == 0) { + const TargetRegisterClass *RC = getInstrOperandRegClass(TRI, TII, II, i); + assert(RC && "Isn't a register operand!"); + VRBase = MRI.createVirtualRegister(RC); + MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + } + + SDValue Op(Node, i); + bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); + } +} + +/// getVR - Return the virtual register corresponding to the specified result +/// of the specified node. +unsigned ScheduleDAGSDNodes::getVR(SDValue Op, + DenseMap &VRBaseMap) { + if (Op.isMachineOpcode() && + Op.getMachineOpcode() == TargetInstrInfo::IMPLICIT_DEF) { + // Add an IMPLICIT_DEF instruction before every use. + unsigned VReg = getDstOfOnlyCopyToRegUse(Op.getNode(), Op.getResNo()); + // IMPLICIT_DEF can produce any type of result so its TargetInstrDesc + // does not include operand register class info. + if (!VReg) { + const TargetRegisterClass *RC = TLI->getRegClassFor(Op.getValueType()); + VReg = MRI.createVirtualRegister(RC); + } + BuildMI(BB, TII->get(TargetInstrInfo::IMPLICIT_DEF), VReg); + return VReg; + } + + DenseMap::iterator I = VRBaseMap.find(Op); + assert(I != VRBaseMap.end() && "Node emitted out of order - late"); + return I->second; +} + + +/// AddOperand - Add the specified operand to the specified machine instr. II +/// specifies the instruction information for the node, and IIOpNum is the +/// operand number (in the II) that we are adding. IIOpNum and II are used for +/// assertions only. +void ScheduleDAGSDNodes::AddOperand(MachineInstr *MI, SDValue Op, + unsigned IIOpNum, + const TargetInstrDesc *II, + DenseMap &VRBaseMap) { + if (Op.isMachineOpcode()) { + // Note that this case is redundant with the final else block, but we + // include it because it is the most common and it makes the logic + // simpler here. + assert(Op.getValueType() != MVT::Other && + Op.getValueType() != MVT::Flag && + "Chain and flag operands should occur at end of operand list!"); + // Get/emit the operand. + unsigned VReg = getVR(Op, VRBaseMap); + const TargetInstrDesc &TID = MI->getDesc(); + bool isOptDef = IIOpNum < TID.getNumOperands() && + TID.OpInfo[IIOpNum].isOptionalDef(); + MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef)); + + // Verify that it is right. + assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); +#ifndef NDEBUG + if (II) { + // There may be no register class for this operand if it is a variadic + // argument (RC will be NULL in this case). In this case, we just assume + // the regclass is ok. + const TargetRegisterClass *RC = + getInstrOperandRegClass(TRI, TII, *II, IIOpNum); + assert((RC || II->isVariadic()) && "Expected reg class info!"); + const TargetRegisterClass *VRC = MRI.getRegClass(VReg); + if (RC && VRC != RC) { + cerr << "Register class of operand and regclass of use don't agree!\n"; + cerr << "Operand = " << IIOpNum << "\n"; + cerr << "Op->Val = "; Op.getNode()->dump(DAG); cerr << "\n"; + cerr << "MI = "; MI->print(cerr); + cerr << "VReg = " << VReg << "\n"; + cerr << "VReg RegClass size = " << VRC->getSize() + << ", align = " << VRC->getAlignment() << "\n"; + cerr << "Expected RegClass size = " << RC->getSize() + << ", align = " << RC->getAlignment() << "\n"; + cerr << "Fatal error, aborting.\n"; + abort(); + } + } +#endif + } else if (ConstantSDNode *C = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateImm(C->getZExtValue())); + } else if (ConstantFPSDNode *F = dyn_cast(Op)) { + const ConstantFP *CFP = F->getConstantFPValue(); + MI->addOperand(MachineOperand::CreateFPImm(CFP)); + } else if (RegisterSDNode *R = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateReg(R->getReg(), false)); + } else if (GlobalAddressSDNode *TGA = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateGA(TGA->getGlobal(),TGA->getOffset())); + } else if (BasicBlockSDNode *BB = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateMBB(BB->getBasicBlock())); + } else if (FrameIndexSDNode *FI = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateFI(FI->getIndex())); + } else if (JumpTableSDNode *JT = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateJTI(JT->getIndex())); + } else if (ConstantPoolSDNode *CP = dyn_cast(Op)) { + int Offset = CP->getOffset(); + unsigned Align = CP->getAlignment(); + const Type *Type = CP->getType(); + // MachineConstantPool wants an explicit alignment. + if (Align == 0) { + Align = TM.getTargetData()->getPreferredTypeAlignmentShift(Type); + if (Align == 0) { + // Alignment of vector types. FIXME! + Align = TM.getTargetData()->getABITypeSize(Type); + Align = Log2_64(Align); + } + } + + unsigned Idx; + if (CP->isMachineConstantPoolEntry()) + Idx = ConstPool->getConstantPoolIndex(CP->getMachineCPVal(), Align); + else + Idx = ConstPool->getConstantPoolIndex(CP->getConstVal(), Align); + MI->addOperand(MachineOperand::CreateCPI(Idx, Offset)); + } else if (ExternalSymbolSDNode *ES = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateES(ES->getSymbol())); + } else { + assert(Op.getValueType() != MVT::Other && + Op.getValueType() != MVT::Flag && + "Chain and flag operands should occur at end of operand list!"); + unsigned VReg = getVR(Op, VRBaseMap); + MI->addOperand(MachineOperand::CreateReg(VReg, false)); + + // Verify that it is right. Note that the reg class of the physreg and the + // vreg don't necessarily need to match, but the target copy insertion has + // to be able to handle it. This handles things like copies from ST(0) to + // an FP vreg on x86. + assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); + if (II && !II->isVariadic()) { + assert(getInstrOperandRegClass(TRI, TII, *II, IIOpNum) && + "Don't have operand info for this instruction!"); + } + } +} + +/// getSubRegisterRegClass - Returns the register class of specified register +/// class' "SubIdx"'th sub-register class. +static const TargetRegisterClass* +getSubRegisterRegClass(const TargetRegisterClass *TRC, unsigned SubIdx) { + // Pick the register class of the subregister + TargetRegisterInfo::regclass_iterator I = + TRC->subregclasses_begin() + SubIdx-1; + assert(I < TRC->subregclasses_end() && + "Invalid subregister index for register class"); + return *I; +} + +/// getSuperRegisterRegClass - Returns the register class of a superreg A whose +/// "SubIdx"'th sub-register class is the specified register class and whose +/// type matches the specified type. +static const TargetRegisterClass* +getSuperRegisterRegClass(const TargetRegisterClass *TRC, + unsigned SubIdx, MVT VT) { + // Pick the register class of the superegister for this type + for (TargetRegisterInfo::regclass_iterator I = TRC->superregclasses_begin(), + E = TRC->superregclasses_end(); I != E; ++I) + if ((*I)->hasType(VT) && getSubRegisterRegClass(*I, SubIdx) == TRC) + return *I; + assert(false && "Couldn't find the register class"); + return 0; +} + +/// EmitSubregNode - Generate machine code for subreg nodes. +/// +void ScheduleDAGSDNodes::EmitSubregNode(SDNode *Node, + DenseMap &VRBaseMap) { + unsigned VRBase = 0; + unsigned Opc = Node->getMachineOpcode(); + + // If the node is only used by a CopyToReg and the dest reg is a vreg, use + // the CopyToReg'd destination register instead of creating a new vreg. + for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); + UI != E; ++UI) { + SDNode *User = *UI; + if (User->getOpcode() == ISD::CopyToReg && + User->getOperand(2).getNode() == Node) { + unsigned DestReg = cast(User->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(DestReg)) { + VRBase = DestReg; + break; + } + } + } + + if (Opc == TargetInstrInfo::EXTRACT_SUBREG) { + unsigned SubIdx = cast(Node->getOperand(1))->getZExtValue(); + + // Create the extract_subreg machine instruction. + MachineInstr *MI = BuildMI(*MF, TII->get(TargetInstrInfo::EXTRACT_SUBREG)); + + // Figure out the register class to create for the destreg. + unsigned VReg = getVR(Node->getOperand(0), VRBaseMap); + const TargetRegisterClass *TRC = MRI.getRegClass(VReg); + const TargetRegisterClass *SRC = getSubRegisterRegClass(TRC, SubIdx); + + if (VRBase) { + // Grab the destination register +#ifndef NDEBUG + const TargetRegisterClass *DRC = MRI.getRegClass(VRBase); + assert(SRC && DRC && SRC == DRC && + "Source subregister and destination must have the same class"); +#endif + } else { + // Create the reg + assert(SRC && "Couldn't find source register class"); + VRBase = MRI.createVirtualRegister(SRC); + } + + // Add def, source, and subreg index + MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap); + MI->addOperand(MachineOperand::CreateImm(SubIdx)); + BB->push_back(MI); + } else if (Opc == TargetInstrInfo::INSERT_SUBREG || + Opc == TargetInstrInfo::SUBREG_TO_REG) { + SDValue N0 = Node->getOperand(0); + SDValue N1 = Node->getOperand(1); + SDValue N2 = Node->getOperand(2); + unsigned SubReg = getVR(N1, VRBaseMap); + unsigned SubIdx = cast(N2)->getZExtValue(); + + + // Figure out the register class to create for the destreg. + const TargetRegisterClass *TRC = 0; + if (VRBase) { + TRC = MRI.getRegClass(VRBase); + } else { + TRC = getSuperRegisterRegClass(MRI.getRegClass(SubReg), SubIdx, + Node->getValueType(0)); + assert(TRC && "Couldn't determine register class for insert_subreg"); + VRBase = MRI.createVirtualRegister(TRC); // Create the reg + } + + // Create the insert_subreg or subreg_to_reg machine instruction. + MachineInstr *MI = BuildMI(*MF, TII->get(Opc)); + MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + + // If creating a subreg_to_reg, then the first input operand + // is an implicit value immediate, otherwise it's a register + if (Opc == TargetInstrInfo::SUBREG_TO_REG) { + const ConstantSDNode *SD = cast(N0); + MI->addOperand(MachineOperand::CreateImm(SD->getZExtValue())); + } else + AddOperand(MI, N0, 0, 0, VRBaseMap); + // Add the subregster being inserted + AddOperand(MI, N1, 0, 0, VRBaseMap); + MI->addOperand(MachineOperand::CreateImm(SubIdx)); + BB->push_back(MI); + } else + assert(0 && "Node is not insert_subreg, extract_subreg, or subreg_to_reg"); + + SDValue Op(Node, 0); + bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); +} + +/// EmitNode - Generate machine code for an node and needed dependencies. +/// +void ScheduleDAGSDNodes::EmitNode(SDNode *Node, bool IsClone, + DenseMap &VRBaseMap) { + // If machine instruction + if (Node->isMachineOpcode()) { + unsigned Opc = Node->getMachineOpcode(); + + // Handle subreg insert/extract specially + if (Opc == TargetInstrInfo::EXTRACT_SUBREG || + Opc == TargetInstrInfo::INSERT_SUBREG || + Opc == TargetInstrInfo::SUBREG_TO_REG) { + EmitSubregNode(Node, VRBaseMap); + return; + } + + if (Opc == TargetInstrInfo::IMPLICIT_DEF) + // We want a unique VR for each IMPLICIT_DEF use. + return; + + const TargetInstrDesc &II = TII->get(Opc); + unsigned NumResults = CountResults(Node); + unsigned NodeOperands = CountOperands(Node); + unsigned MemOperandsEnd = ComputeMemOperandsEnd(Node); + bool HasPhysRegOuts = (NumResults > II.getNumDefs()) && + II.getImplicitDefs() != 0; +#ifndef NDEBUG + unsigned NumMIOperands = NodeOperands + NumResults; + assert((II.getNumOperands() == NumMIOperands || + HasPhysRegOuts || II.isVariadic()) && + "#operands for dag node doesn't match .td file!"); +#endif + + // Create the new machine instruction. + MachineInstr *MI = BuildMI(*MF, II); + + // Add result register values for things that are defined by this + // instruction. + if (NumResults) + CreateVirtualRegisters(Node, MI, II, VRBaseMap); + + // Emit all of the actual operands of this instruction, adding them to the + // instruction as appropriate. + for (unsigned i = 0; i != NodeOperands; ++i) + AddOperand(MI, Node->getOperand(i), i+II.getNumDefs(), &II, VRBaseMap); + + // Emit all of the memory operands of this instruction + for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i) + AddMemOperand(MI, cast(Node->getOperand(i))->MO); + + // Commute node if it has been determined to be profitable. + if (CommuteSet.count(Node)) { + MachineInstr *NewMI = TII->commuteInstruction(MI); + if (NewMI == 0) + DOUT << "Sched: COMMUTING FAILED!\n"; + else { + DOUT << "Sched: COMMUTED TO: " << *NewMI; + if (MI != NewMI) { + MF->DeleteMachineInstr(MI); + MI = NewMI; + } + ++NumCommutes; + } + } + + if (II.usesCustomDAGSchedInsertionHook()) + // Insert this instruction into the basic block using a target + // specific inserter which may returns a new basic block. + BB = TLI->EmitInstrWithCustomInserter(MI, BB); + else + BB->push_back(MI); + + // Additional results must be an physical register def. + if (HasPhysRegOuts) { + for (unsigned i = II.getNumDefs(); i < NumResults; ++i) { + unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()]; + if (Node->hasAnyUseOfValue(i)) + EmitCopyFromReg(Node, i, IsClone, Reg, VRBaseMap); + } + } + return; + } + + switch (Node->getOpcode()) { + default: +#ifndef NDEBUG + Node->dump(DAG); +#endif + assert(0 && "This target-independent node should have been selected!"); + break; + case ISD::EntryToken: + assert(0 && "EntryToken should have been excluded from the schedule!"); + break; + case ISD::TokenFactor: // fall thru + break; + case ISD::CopyToReg: { + unsigned SrcReg; + SDValue SrcVal = Node->getOperand(2); + if (RegisterSDNode *R = dyn_cast(SrcVal)) + SrcReg = R->getReg(); + else + SrcReg = getVR(SrcVal, VRBaseMap); + + unsigned DestReg = cast(Node->getOperand(1))->getReg(); + if (SrcReg == DestReg) // Coalesced away the copy? Ignore. + break; + + const TargetRegisterClass *SrcTRC = 0, *DstTRC = 0; + // Get the register classes of the src/dst. + if (TargetRegisterInfo::isVirtualRegister(SrcReg)) + SrcTRC = MRI.getRegClass(SrcReg); + else + SrcTRC = TRI->getPhysicalRegisterRegClass(SrcReg,SrcVal.getValueType()); + + if (TargetRegisterInfo::isVirtualRegister(DestReg)) + DstTRC = MRI.getRegClass(DestReg); + else + DstTRC = TRI->getPhysicalRegisterRegClass(DestReg, + Node->getOperand(1).getValueType()); + TII->copyRegToReg(*BB, BB->end(), DestReg, SrcReg, DstTRC, SrcTRC); + break; + } + case ISD::CopyFromReg: { + unsigned SrcReg = cast(Node->getOperand(1))->getReg(); + EmitCopyFromReg(Node, 0, IsClone, SrcReg, VRBaseMap); + break; + } + case ISD::INLINEASM: { + unsigned NumOps = Node->getNumOperands(); + if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag) + --NumOps; // Ignore the flag operand. + + // Create the inline asm machine instruction. + MachineInstr *MI = BuildMI(*MF, TII->get(TargetInstrInfo::INLINEASM)); + + // Add the asm string as an external symbol operand. + const char *AsmStr = + cast(Node->getOperand(1))->getSymbol(); + MI->addOperand(MachineOperand::CreateES(AsmStr)); + + // Add all of the operand registers to the instruction. + for (unsigned i = 2; i != NumOps;) { + unsigned Flags = + cast(Node->getOperand(i))->getZExtValue(); + unsigned NumVals = Flags >> 3; + + MI->addOperand(MachineOperand::CreateImm(Flags)); + ++i; // Skip the ID value. + + switch (Flags & 7) { + default: assert(0 && "Bad flags!"); + case 2: // Def of register. + for (; NumVals; --NumVals, ++i) { + unsigned Reg = cast(Node->getOperand(i))->getReg(); + MI->addOperand(MachineOperand::CreateReg(Reg, true)); + } + break; + case 6: // Def of earlyclobber register. + for (; NumVals; --NumVals, ++i) { + unsigned Reg = cast(Node->getOperand(i))->getReg(); + MI->addOperand(MachineOperand::CreateReg(Reg, true, false, false, + false, 0, true)); + } + break; + case 1: // Use of register. + case 3: // Immediate. + case 4: // Addressing mode. + // The addressing mode has been selected, just add all of the + // operands to the machine instruction. + for (; NumVals; --NumVals, ++i) + AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap); + break; + } + } + BB->push_back(MI); + break; + } + } +} + +/// EmitSchedule - Emit the machine code in scheduled order. +MachineBasicBlock *ScheduleDAGSDNodes::EmitSchedule() { + DenseMap VRBaseMap; + DenseMap CopyVRBaseMap; + for (unsigned i = 0, e = Sequence.size(); i != e; i++) { + SUnit *SU = Sequence[i]; + if (!SU) { + // Null SUnit* is a noop. + EmitNoop(); + continue; + } + + // For pre-regalloc scheduling, create instructions corresponding to the + // SDNode and any flagged SDNodes and append them to the block. + SmallVector FlaggedNodes; + for (SDNode *N = SU->getNode()->getFlaggedNode(); N; N = N->getFlaggedNode()) + FlaggedNodes.push_back(N); + while (!FlaggedNodes.empty()) { + EmitNode(FlaggedNodes.back(), SU->OrigNode != SU, VRBaseMap); + FlaggedNodes.pop_back(); + } + if (!SU->getNode()) + EmitCrossRCCopy(SU, CopyVRBaseMap); + else + EmitNode(SU->getNode(), SU->OrigNode != SU, VRBaseMap); + } + + return BB; +} diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 23822fa7fb1..e43b32535c3 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -34,7 +34,7 @@ #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/Target/TargetRegisterInfo.h" diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp index d7481085dfd..03b78c33efb 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp @@ -15,7 +15,7 @@ #include "llvm/Function.h" #include "llvm/Assembly/Writer.h" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" @@ -383,71 +383,7 @@ void SelectionDAG::setSubgraphColor(SDNode *N, const char *Color) { #endif } -namespace llvm { - template<> - struct DOTGraphTraits : public DefaultDOTGraphTraits { - static std::string getGraphName(const ScheduleDAG *G) { - return G->MF->getFunction()->getName(); - } - - static bool renderGraphFromBottomUp() { - return true; - } - - static bool hasNodeAddressLabel(const SUnit *Node, - const ScheduleDAG *Graph) { - return true; - } - - /// If you want to override the dot attributes printed for a particular - /// edge, override this method. - template - static std::string getEdgeAttributes(const void *Node, EdgeIter EI) { - if (EI.isSpecialDep()) - return "color=cyan,style=dashed"; - if (EI.isCtrlDep()) - return "color=blue,style=dashed"; - return ""; - } - - - static std::string getNodeLabel(const SUnit *Node, - const ScheduleDAG *Graph); - static std::string getNodeAttributes(const SUnit *N, - const ScheduleDAG *Graph) { - return "shape=Mrecord"; - } - - static void addCustomGraphFeatures(ScheduleDAG *G, - GraphWriter &GW) { - // Draw a special "GraphRoot" node to indicate the root of the graph. - GW.emitSimpleNode(0, "plaintext=circle", "GraphRoot"); - if (G->DAG) { - // For an SDNode-based ScheduleDAG, point to the root of the ScheduleDAG. - const SDNode *N = G->DAG->getRoot().getNode(); - if (N && N->getNodeId() != -1) - GW.emitEdge(0, -1, &G->SUnits[N->getNodeId()], -1, - "color=blue,style=dashed"); - } else { - // For a MachineInstr-based ScheduleDAG, find a root to point to. - for (unsigned i = 0, e = G->SUnits.size(); i != e; ++i) { - if (G->SUnits[i].Succs.empty()) { - GW.emitEdge(0, -1, &G->SUnits[i], -1, - "color=blue,style=dashed"); - break; - } - } - } - } - }; -} - -std::string DOTGraphTraits::getNodeLabel(const SUnit *SU, - const ScheduleDAG *G) { - return G->getGraphNodeLabel(SU); -} - -std::string ScheduleDAG::getGraphNodeLabel(const SUnit *SU) const { +std::string ScheduleDAGSDNodes::getGraphNodeLabel(const SUnit *SU) const { std::string s; raw_string_ostream O(s); O << "SU(" << SU->NodeNum << "): "; @@ -467,17 +403,13 @@ std::string ScheduleDAG::getGraphNodeLabel(const SUnit *SU) const { return O.str(); } -/// viewGraph - Pop up a ghostview window with the reachable parts of the DAG -/// rendered using 'dot'. -/// -void ScheduleDAG::viewGraph() { -// This code is only for debugging! -#ifndef NDEBUG - ViewGraph(this, "dag." + MF->getFunction()->getName(), - "Scheduling-Units Graph for " + MF->getFunction()->getName() + ':' + - BB->getBasicBlock()->getName()); -#else - cerr << "ScheduleDAG::viewGraph is only available in debug builds on " - << "systems with Graphviz or gv!\n"; -#endif // NDEBUG +void ScheduleDAGSDNodes::getCustomGraphFeatures(GraphWriter &GW) const { + if (DAG) { + // Draw a special "GraphRoot" node to indicate the root of the graph. + GW.emitSimpleNode(0, "plaintext=circle", "GraphRoot"); + const SDNode *N = DAG->getRoot().getNode(); + if (N && N->getNodeId() != -1) + GW.emitEdge(0, -1, &SUnits[N->getNodeId()], -1, + "color=blue,style=dashed"); + } } diff --git a/lib/Target/CellSPU/SPUHazardRecognizers.h b/lib/Target/CellSPU/SPUHazardRecognizers.h index 8d74c5ccb76..6b73083bd1e 100644 --- a/lib/Target/CellSPU/SPUHazardRecognizers.h +++ b/lib/Target/CellSPU/SPUHazardRecognizers.h @@ -15,7 +15,7 @@ #ifndef SPUHAZRECS_H #define SPUHAZRECS_H -#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" #include "SPUInstrInfo.h" namespace llvm { diff --git a/lib/Target/PowerPC/PPCHazardRecognizers.h b/lib/Target/PowerPC/PPCHazardRecognizers.h index 7ba2e6e7130..8957d180da0 100644 --- a/lib/Target/PowerPC/PPCHazardRecognizers.h +++ b/lib/Target/PowerPC/PPCHazardRecognizers.h @@ -14,7 +14,7 @@ #ifndef PPCHAZRECS_H #define PPCHAZRECS_H -#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/ScheduleDAGSDNodes.h" #include "PPCInstrInfo.h" namespace llvm {