Order ///< Any other ordering dependency.
};
+ enum OrderKind {
+ Barrier, ///< An unknown scheduling barrier.
+ MayAliasMem, ///< Nonvolatile load/Store instructions that may alias.
+ MustAliasMem, ///< Nonvolatile load/Store instructions that must alias.
+ Artificial ///< Arbitrary weak DAG edge (no actual dependence).
+ };
+
private:
/// Dep - A pointer to the depending/depended-on SUnit, and an enum
/// indicating the kind of the dependency.
unsigned Reg;
/// Order - Additional information about Order dependencies.
- struct {
- /// isNormalMemory - True if both sides of the dependence
- /// access memory in non-volatile and fully modeled ways.
- bool isNormalMemory : 1;
-
- /// isMustAlias - True if both sides of the dependence are known to
- /// access the same memory.
- bool isMustAlias : 1;
-
- /// isArtificial - True if this is an artificial dependency, meaning
- /// it is not necessary for program correctness, and may be safely
- /// deleted if necessary.
- bool isArtificial : 1;
- } Order;
+ unsigned OrdKind; // enum OrderKind
} Contents;
/// Latency - The time associated with this edge. Often this is just
/// models may provide additional information about specific edges.
unsigned Latency;
/// Record MinLatency seperately from "expected" Latency.
+ ///
+ /// FIXME: this field is not packed on LP64. Convert to 16-bit DAG edge
+ /// latency after introducing saturating truncation.
unsigned MinLatency;
public:
SDep() : Dep(0, Data) {}
/// SDep - Construct an SDep with the specified values.
- SDep(SUnit *S, Kind kind, unsigned latency = 1, unsigned Reg = 0,
- bool isNormalMemory = false, bool isMustAlias = false,
- bool isArtificial = false)
- : Dep(S, kind), Contents(), Latency(latency), MinLatency(latency) {
+ SDep(SUnit *S, Kind kind, unsigned Reg)
+ : Dep(S, kind), Contents() {
switch (kind) {
+ default:
+ llvm_unreachable("Reg given for non-register dependence!");
case Anti:
case Output:
assert(Reg != 0 &&
"SDep::Anti and SDep::Output must use a non-zero Reg!");
- // fall through
- case Data:
- assert(!isMustAlias && "isMustAlias only applies with SDep::Order!");
- assert(!isArtificial && "isArtificial only applies with SDep::Order!");
Contents.Reg = Reg;
+ Latency = 0;
break;
- case Order:
- assert(Reg == 0 && "Reg given for non-register dependence!");
- Contents.Order.isNormalMemory = isNormalMemory;
- Contents.Order.isMustAlias = isMustAlias;
- Contents.Order.isArtificial = isArtificial;
+ case Data:
+ Contents.Reg = Reg;
+ Latency = 1;
break;
}
+ MinLatency = Latency;
+ }
+ SDep(SUnit *S, OrderKind kind)
+ : Dep(S, Order), Contents(), Latency(0), MinLatency(0) {
+ Contents.OrdKind = kind;
}
/// Return true if the specified SDep is equivalent except for latency.
case Output:
return Contents.Reg == Other.Contents.Reg;
case Order:
- return Contents.Order.isNormalMemory ==
- Other.Contents.Order.isNormalMemory &&
- Contents.Order.isMustAlias == Other.Contents.Order.isMustAlias &&
- Contents.Order.isArtificial == Other.Contents.Order.isArtificial;
+ return Contents.OrdKind == Other.Contents.OrdKind;
}
llvm_unreachable("Invalid dependency kind!");
}
/// memory accesses where both sides of the dependence access memory
/// in non-volatile and fully modeled ways.
bool isNormalMemory() const {
- return getKind() == Order && Contents.Order.isNormalMemory;
+ return getKind() == Order && (Contents.OrdKind == MayAliasMem
+ || Contents.OrdKind == MustAliasMem);
}
/// isMustAlias - Test if this is an Order dependence that is marked
/// as "must alias", meaning that the SUnits at either end of the edge
/// have a memory dependence on a known memory location.
bool isMustAlias() const {
- return getKind() == Order && Contents.Order.isMustAlias;
+ return getKind() == Order && Contents.OrdKind == MustAliasMem;
}
/// isArtificial - Test if this is an Order dependence that is marked
/// as "artificial", meaning it isn't necessary for correctness.
bool isArtificial() const {
- return getKind() == Order && Contents.Order.isArtificial;
+ return getKind() == Order && Contents.OrdKind == Artificial;
}
/// isAssignedRegDep - Test if this is a Data dependence that is
if (UseSU == SU)
continue;
- SDep dep(SU, SDep::Data, 1, *Alias);
+ SDep dep(SU, SDep::Data, *Alias);
// Adjust the dependence latency using operand def/use information,
// then allow the target to perform its own adjustments.
(Kind != SDep::Output || !MO.isDead() ||
!DefSU->getInstr()->registerDefIsDead(*Alias))) {
if (Kind == SDep::Anti)
- DefSU->addPred(SDep(SU, Kind, 0, /*Reg=*/*Alias));
+ DefSU->addPred(SDep(SU, Kind, /*Reg=*/*Alias));
else {
- unsigned AOLat =
+ SDep Dep(SU, Kind, /*Reg=*/*Alias);
+ unsigned OutLatency =
SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr());
- DefSU->addPred(SDep(SU, Kind, AOLat, /*Reg=*/*Alias));
+ Dep.setMinLatency(OutLatency);
+ Dep.setLatency(OutLatency);
+ DefSU->addPred(Dep);
}
}
}
else {
SUnit *DefSU = DefI->SU;
if (DefSU != SU && DefSU != &ExitSU) {
+ SDep Dep(SU, SDep::Output, Reg);
unsigned OutLatency =
SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr());
- DefSU->addPred(SDep(SU, SDep::Output, OutLatency, Reg));
+ Dep.setMinLatency(OutLatency);
+ Dep.setLatency(OutLatency);
+ DefSU->addPred(Dep);
}
DefI->SU = SU;
}
if (DefSU) {
// The reaching Def lives within this scheduling region.
// Create a data dependence.
- SDep dep(DefSU, SDep::Data, 1, Reg);
+ SDep dep(DefSU, SDep::Data, Reg);
// Adjust the dependence latency using operand def/use information, then
// allow the target to perform its own adjustments.
int DefOp = Def->findRegisterDefOperandIdx(Reg);
// Add antidependence to the following def of the vreg it uses.
VReg2SUnitMap::iterator DefI = VRegDefs.find(Reg);
if (DefI != VRegDefs.end() && DefI->SU != SU)
- DefI->SU->addPred(SDep(SU, SDep::Anti, 0, Reg));
+ DefI->SU->addPred(SDep(SU, SDep::Anti, Reg));
}
/// Return true if MI is an instruction we are unable to reason about
// and stop descending.
if (*Depth > 200 ||
MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr())) {
- SUb->addPred(SDep(SUa, SDep::Order, /*Latency=*/0, /*Reg=*/0,
- /*isNormalMemory=*/true));
+ SUb->addPred(SDep(SUa, SDep::MayAliasMem));
return *Depth;
}
// Track current depth.
if (SU == *I)
continue;
if (MIsNeedChainEdge(AA, MFI, SU->getInstr(), (*I)->getInstr())) {
- unsigned Latency = ((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0;
- (*I)->addPred(SDep(SU, SDep::Order, Latency, /*Reg=*/0,
- /*isNormalMemory=*/true));
+ SDep Dep(SU, SDep::MayAliasMem);
+ Dep.setLatency(((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0);
+ (*I)->addPred(Dep);
}
// Now go through all the chain successors and iterate from them.
// Keep track of visited nodes.
// If this is a false dependency,
// do not add the edge, but rememeber the rejected node.
if (!EnableAASchedMI ||
- MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr()))
- SUb->addPred(SDep(SUa, SDep::Order, TrueMemOrderLatency, /*Reg=*/0,
- isNormalMemory));
+ MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr())) {
+ SDep Dep(SUa, isNormalMemory ? SDep::MayAliasMem : SDep::Barrier);
+ Dep.setLatency(TrueMemOrderLatency);
+ SUb->addPred(Dep);
+ }
else {
// Duplicate entries should be ignored.
RejectList.insert(SUb);
// references, even those that are known to not alias.
for (std::map<const Value *, SUnit *>::iterator I =
NonAliasMemDefs.begin(), E = NonAliasMemDefs.end(); I != E; ++I) {
- I->second->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+ I->second->addPred(SDep(SU, SDep::Barrier));
}
for (std::map<const Value *, std::vector<SUnit *> >::iterator I =
NonAliasMemUses.begin(), E = NonAliasMemUses.end(); I != E; ++I) {
- for (unsigned i = 0, e = I->second.size(); i != e; ++i)
- I->second[i]->addPred(SDep(SU, SDep::Order, TrueMemOrderLatency));
+ for (unsigned i = 0, e = I->second.size(); i != e; ++i) {
+ SDep Dep(SU, SDep::Barrier);
+ Dep.setLatency(TrueMemOrderLatency);
+ I->second[i]->addPred(Dep);
+ }
}
// Add SU to the barrier chain.
if (BarrierChain)
- BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+ BarrierChain->addPred(SDep(SU, SDep::Barrier));
BarrierChain = SU;
// This is a barrier event that acts as a pivotal node in the DAG,
// so it is safe to clear list of exposed nodes.
// SU and barrier _could_ be reordered, they should not. In addition,
// we have lost all RejectMemNodes below barrier.
if (BarrierChain)
- BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+ BarrierChain->addPred(SDep(SU, SDep::Barrier));
} else {
// Treat all other stores conservatively.
goto new_alias_chain;
if (!ExitSU.isPred(SU))
// Push store's up a bit to avoid them getting in between cmp
// and branches.
- ExitSU.addPred(SDep(SU, SDep::Order, 0,
- /*Reg=*/0, /*isNormalMemory=*/false,
- /*isMustAlias=*/false,
- /*isArtificial=*/true));
+ ExitSU.addPred(SDep(SU, SDep::Artificial));
} else if (MI->mayLoad()) {
bool MayAlias = true;
if (MI->isInvariantLoad(AA)) {
if (MayAlias && AliasChain)
addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes);
if (BarrierChain)
- BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+ BarrierChain->addPred(SDep(SU, SDep::Barrier));
}
}
}
}
}
if (isNewLoad) {
- AddPred(NewSU, SDep(LoadSU, SDep::Order, LoadSU->Latency));
+ SDep D(LoadSU, SDep::Barrier);
+ D.setLatency(LoadSU->Latency);
+ AddPred(NewSU, D);
}
++NumUnfolds;
for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
RemovePred(DelDeps[i].first, DelDeps[i].second);
}
-
- AddPred(CopyFromSU, SDep(SU, SDep::Data, SU->Latency, Reg));
- AddPred(CopyToSU, SDep(CopyFromSU, SDep::Data, CopyFromSU->Latency, 0));
+ SDep FromDep(SU, SDep::Data, Reg);
+ FromDep.setLatency(SU->Latency);
+ AddPred(CopyFromSU, FromDep);
+ SDep ToDep(CopyFromSU, SDep::Data, 0);
+ ToDep.setLatency(CopyFromSU->Latency);
+ AddPred(CopyToSU, ToDep);
Copies.push_back(CopyFromSU);
Copies.push_back(CopyToSU);
InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
DEBUG(dbgs() << "Adding an edge from SU # " << TrySU->NodeNum
<< " to SU #" << Copies.front()->NodeNum << "\n");
- AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1,
- /*Reg=*/0, /*isNormalMemory=*/false,
- /*isMustAlias=*/false, /*isArtificial=*/true));
+ AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
NewDef = Copies.back();
}
DEBUG(dbgs() << "Adding an edge from SU # " << NewDef->NodeNum
<< " to SU #" << TrySU->NodeNum << "\n");
LiveRegDefs[Reg] = NewDef;
- AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1,
- /*Reg=*/0, /*isNormalMemory=*/false,
- /*isMustAlias=*/false, /*isArtificial=*/true));
+ AddPred(NewDef, SDep(TrySU, SDep::Artificial));
TrySU->isAvailable = false;
CurSU = NewDef;
}
// Add a data dependency to reflect that NewSU reads the value defined
// by LoadSU.
- AddPred(NewSU, SDep(LoadSU, SDep::Data, LoadSU->Latency));
+ SDep D(LoadSU, SDep::Data, 0);
+ D.setLatency(LoadSU->Latency);
+ AddPred(NewSU, D);
if (isNewLoad)
AvailableQueue->addNode(LoadSU);
// Avoid scheduling the def-side copy before other successors. Otherwise
// we could introduce another physreg interference on the copy and
// continue inserting copies indefinitely.
- SDep D(CopyFromSU, SDep::Order, /*Latency=*/0,
- /*Reg=*/0, /*isNormalMemory=*/false,
- /*isMustAlias=*/false, /*isArtificial=*/true);
- AddPred(SuccSU, D);
+ AddPred(SuccSU, SDep(CopyFromSU, SDep::Artificial));
}
}
for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
RemovePred(DelDeps[i].first, DelDeps[i].second);
- AddPred(CopyFromSU, SDep(SU, SDep::Data, SU->Latency, Reg));
- AddPred(CopyToSU, SDep(CopyFromSU, SDep::Data, CopyFromSU->Latency, 0));
+ SDep FromDep(SU, SDep::Data, Reg);
+ FromDep.setLatency(SU->Latency);
+ AddPred(CopyFromSU, FromDep);
+ SDep ToDep(CopyFromSU, SDep::Data, 0);
+ ToDep.setLatency(CopyFromSU->Latency);
+ AddPred(CopyToSU, ToDep);
AvailableQueue->updateNode(SU);
AvailableQueue->addNode(CopyFromSU);
if (!BtSU->isPending)
AvailableQueue->remove(BtSU);
}
- AddPred(TrySU, SDep(BtSU, SDep::Order, /*Latency=*/1,
- /*Reg=*/0, /*isNormalMemory=*/false,
- /*isMustAlias=*/false, /*isArtificial=*/true));
+ AddPred(TrySU, SDep(BtSU, SDep::Artificial));
// If one or more successors has been unscheduled, then the current
// node is no longer avaialable. Schedule a successor that's now
InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum
<< " to SU #" << Copies.front()->NodeNum << "\n");
- AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1,
- /*Reg=*/0, /*isNormalMemory=*/false,
- /*isMustAlias=*/false,
- /*isArtificial=*/true));
+ AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
NewDef = Copies.back();
}
DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum
<< " to SU #" << TrySU->NodeNum << "\n");
LiveRegDefs[Reg] = NewDef;
- AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1,
- /*Reg=*/0, /*isNormalMemory=*/false,
- /*isMustAlias=*/false,
- /*isArtificial=*/true));
+ AddPred(NewDef, SDep(TrySU, SDep::Artificial));
TrySU->isAvailable = false;
CurSU = NewDef;
}
!scheduleDAG->IsReachable(SuccSU, SU)) {
DEBUG(dbgs() << " Adding a pseudo-two-addr edge from SU #"
<< SU->NodeNum << " to SU #" << SuccSU->NodeNum << "\n");
- scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Order, /*Latency=*/0,
- /*Reg=*/0, /*isNormalMemory=*/false,
- /*isMustAlias=*/false,
- /*isArtificial=*/true));
+ scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Artificial));
}
}
}
if(isChain && OpN->getOpcode() == ISD::TokenFactor)
OpLatency = 0;
- const SDep &dep = SDep(OpSU, isChain ? SDep::Order : SDep::Data,
- OpLatency, PhysReg);
+ SDep Dep = isChain ? SDep(OpSU, SDep::Barrier)
+ : SDep(OpSU, SDep::Data, PhysReg);
+ Dep.setLatency(OpLatency);
if (!isChain && !UnitLatencies) {
- computeOperandLatency(OpN, N, i, const_cast<SDep &>(dep));
- ST.adjustSchedDependency(OpSU, SU, const_cast<SDep &>(dep));
+ computeOperandLatency(OpN, N, i, Dep);
+ ST.adjustSchedDependency(OpSU, SU, Dep);
}
- if (!SU->addPred(dep) && !dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
+ if (!SU->addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
// Multiple register uses are combined in the same SUnit. For example,
// we could have a set of glued nodes with all their defs consumed by
// another set of glued nodes. Register pressure tracking sees this as
LastSequentialCall = &(SUnits[su]);
// Look for a compare that defines a predicate.
else if (SUnits[su].getInstr()->isCompare() && LastSequentialCall)
- SUnits[su].addPred(SDep(LastSequentialCall, SDep::Order, 0, /*Reg=*/0,
- false));
+ SUnits[su].addPred(SDep(LastSequentialCall, SDep::Barrier));
}
}
projects / oota-llvm.git / commitdiff