//===----------------------------------------------------------------------===//
#include "LegalizeTypes.h"
-#include "llvm/CallingConv.h"
-#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/IR/CallingConv.h"
+#include "llvm/IR/DataLayout.h"
#include "llvm/Support/CommandLine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
using namespace llvm;
+#define DEBUG_TYPE "legalize-types"
+
static cl::opt<bool>
EnableExpensiveChecks("enable-legalize-types-checking", cl::Hidden);
// The final node obtained by mapping by ReplacedValues is not marked NewNode.
// Note that ReplacedValues should be applied iteratively.
- // Note that the ReplacedValues map may also map deleted nodes. By iterating
- // over the DAG we only consider non-deleted nodes.
+ // Note that the ReplacedValues map may also map deleted nodes (by iterating
+ // over the DAG we never dereference deleted nodes). This means that it may
+ // also map nodes marked NewNode if the deallocated memory was reallocated as
+ // another node, and that new node was not seen by the LegalizeTypes machinery
+ // (for example because it was created but not used). In general, we cannot
+ // distinguish between new nodes and deleted nodes.
SmallVector<SDNode*, 16> NewNodes;
- for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
- E = DAG.allnodes_end(); I != E; ++I) {
+ for (SDNode &Node : DAG.allnodes()) {
// Remember nodes marked NewNode - they are subject to extra checking below.
- if (I->getNodeId() == NewNode)
- NewNodes.push_back(I);
+ if (Node.getNodeId() == NewNode)
+ NewNodes.push_back(&Node);
- for (unsigned i = 0, e = I->getNumValues(); i != e; ++i) {
- SDValue Res(I, i);
+ for (unsigned i = 0, e = Node.getNumValues(); i != e; ++i) {
+ SDValue Res(&Node, i);
bool Failed = false;
unsigned Mapped = 0;
if (ReplacedValues.find(Res) != ReplacedValues.end()) {
Mapped |= 1;
// Check that remapped values are only used by nodes marked NewNode.
- for (SDNode::use_iterator UI = I->use_begin(), UE = I->use_end();
+ for (SDNode::use_iterator UI = Node.use_begin(), UE = Node.use_end();
UI != UE; ++UI)
- if (UI.getUse().getSDValue().getResNo() == i)
+ if (UI.getUse().getResNo() == i)
assert(UI->getNodeId() == NewNode &&
"Remapped value has non-trivial use!");
if (WidenedVectors.find(Res) != WidenedVectors.end())
Mapped |= 128;
- if (I->getNodeId() != Processed) {
- if (Mapped != 0) {
- cerr << "Unprocessed value in a map!";
+ if (Node.getNodeId() != Processed) {
+ // Since we allow ReplacedValues to map deleted nodes, it may map nodes
+ // marked NewNode too, since a deleted node may have been reallocated as
+ // another node that has not been seen by the LegalizeTypes machinery.
+ if ((Node.getNodeId() == NewNode && Mapped > 1) ||
+ (Node.getNodeId() != NewNode && Mapped != 0)) {
+ dbgs() << "Unprocessed value in a map!";
Failed = true;
}
- } else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(I)) {
- // FIXME: Because of PR2957, the build vector can be placed on this
- // list but if the associated vector shuffle is split, the build vector
- // can also be split so we allow this to go through for now.
- if (Mapped > 1 && Res.getOpcode() != ISD::BUILD_VECTOR) {
- cerr << "Value with legal type was transformed!";
+ } else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(&Node)) {
+ if (Mapped > 1) {
+ dbgs() << "Value with legal type was transformed!";
Failed = true;
}
} else {
if (Mapped == 0) {
- cerr << "Processed value not in any map!";
+ dbgs() << "Processed value not in any map!";
Failed = true;
} else if (Mapped & (Mapped - 1)) {
- cerr << "Value in multiple maps!";
+ dbgs() << "Value in multiple maps!";
Failed = true;
}
}
if (Failed) {
if (Mapped & 1)
- cerr << " ReplacedValues";
+ dbgs() << " ReplacedValues";
if (Mapped & 2)
- cerr << " PromotedIntegers";
+ dbgs() << " PromotedIntegers";
if (Mapped & 4)
- cerr << " SoftenedFloats";
+ dbgs() << " SoftenedFloats";
if (Mapped & 8)
- cerr << " ScalarizedVectors";
+ dbgs() << " ScalarizedVectors";
if (Mapped & 16)
- cerr << " ExpandedIntegers";
+ dbgs() << " ExpandedIntegers";
if (Mapped & 32)
- cerr << " ExpandedFloats";
+ dbgs() << " ExpandedFloats";
if (Mapped & 64)
- cerr << " SplitVectors";
+ dbgs() << " SplitVectors";
if (Mapped & 128)
- cerr << " WidenedVectors";
- cerr << "\n";
- abort();
+ dbgs() << " WidenedVectors";
+ dbgs() << "\n";
+ llvm_unreachable(nullptr);
}
}
}
// Walk all nodes in the graph, assigning them a NodeId of 'ReadyToProcess'
// (and remembering them) if they are leaves and assigning 'Unanalyzed' if
// non-leaves.
- for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
- E = DAG.allnodes_end(); I != E; ++I) {
- if (I->getNumOperands() == 0) {
- I->setNodeId(ReadyToProcess);
- Worklist.push_back(I);
+ for (SDNode &Node : DAG.allnodes()) {
+ if (Node.getNumOperands() == 0) {
+ Node.setNodeId(ReadyToProcess);
+ Worklist.push_back(&Node);
} else {
- I->setNodeId(Unanalyzed);
+ Node.setNodeId(Unanalyzed);
}
}
// Scan the values produced by the node, checking to see if any result
// types are illegal.
for (unsigned i = 0, NumResults = N->getNumValues(); i < NumResults; ++i) {
- MVT ResultVT = N->getValueType(i);
+ EVT ResultVT = N->getValueType(i);
switch (getTypeAction(ResultVT)) {
- default:
- assert(false && "Unknown action!");
- case Legal:
+ case TargetLowering::TypeLegal:
break;
// The following calls must take care of *all* of the node's results,
// not just the illegal result they were passed (this includes results
// with a legal type). Results can be remapped using ReplaceValueWith,
// or their promoted/expanded/etc values registered in PromotedIntegers,
// ExpandedIntegers etc.
- case PromoteInteger:
+ case TargetLowering::TypePromoteInteger:
PromoteIntegerResult(N, i);
Changed = true;
goto NodeDone;
- case ExpandInteger:
+ case TargetLowering::TypeExpandInteger:
ExpandIntegerResult(N, i);
Changed = true;
goto NodeDone;
- case SoftenFloat:
+ case TargetLowering::TypeSoftenFloat:
SoftenFloatResult(N, i);
Changed = true;
goto NodeDone;
- case ExpandFloat:
+ case TargetLowering::TypeExpandFloat:
ExpandFloatResult(N, i);
Changed = true;
goto NodeDone;
- case ScalarizeVector:
+ case TargetLowering::TypeScalarizeVector:
ScalarizeVectorResult(N, i);
Changed = true;
goto NodeDone;
- case SplitVector:
+ case TargetLowering::TypeSplitVector:
SplitVectorResult(N, i);
Changed = true;
goto NodeDone;
- case WidenVector:
+ case TargetLowering::TypeWidenVector:
WidenVectorResult(N, i);
Changed = true;
goto NodeDone;
+ case TargetLowering::TypePromoteFloat:
+ PromoteFloatResult(N, i);
+ Changed = true;
+ goto NodeDone;
}
}
if (IgnoreNodeResults(N->getOperand(i).getNode()))
continue;
- if (N->getOpcode() == ISD::VECTOR_SHUFFLE && i == 2) {
- // The shuffle mask doesn't need to be a legal vector type.
- // FIXME: We can remove this once we fix PR2957.
- SetIgnoredNodeResult(N->getOperand(2).getNode());
- continue;
- }
-
- MVT OpVT = N->getOperand(i).getValueType();
+ EVT OpVT = N->getOperand(i).getValueType();
switch (getTypeAction(OpVT)) {
- default:
- assert(false && "Unknown action!");
- case Legal:
+ case TargetLowering::TypeLegal:
continue;
// The following calls must either replace all of the node's results
// using ReplaceValueWith, and return "false"; or update the node's
// operands in place, and return "true".
- case PromoteInteger:
+ case TargetLowering::TypePromoteInteger:
NeedsReanalyzing = PromoteIntegerOperand(N, i);
Changed = true;
break;
- case ExpandInteger:
+ case TargetLowering::TypeExpandInteger:
NeedsReanalyzing = ExpandIntegerOperand(N, i);
Changed = true;
break;
- case SoftenFloat:
+ case TargetLowering::TypeSoftenFloat:
NeedsReanalyzing = SoftenFloatOperand(N, i);
Changed = true;
break;
- case ExpandFloat:
+ case TargetLowering::TypeExpandFloat:
NeedsReanalyzing = ExpandFloatOperand(N, i);
Changed = true;
break;
- case ScalarizeVector:
+ case TargetLowering::TypeScalarizeVector:
NeedsReanalyzing = ScalarizeVectorOperand(N, i);
Changed = true;
break;
- case SplitVector:
+ case TargetLowering::TypeSplitVector:
NeedsReanalyzing = SplitVectorOperand(N, i);
Changed = true;
break;
- case WidenVector:
+ case TargetLowering::TypeWidenVector:
NeedsReanalyzing = WidenVectorOperand(N, i);
Changed = true;
break;
+ case TargetLowering::TypePromoteFloat:
+ NeedsReanalyzing = PromoteFloatOperand(N, i);
+ Changed = true;
+ break;
}
break;
}
// The node morphed - this is equivalent to legalizing by replacing every
// value of N with the corresponding value of M. So do that now.
- N->setNodeId(ReadyToProcess);
assert(N->getNumValues() == M->getNumValues() &&
"Node morphing changed the number of results!");
for (unsigned i = 0, e = N->getNumValues(); i != e; ++i)
// Replacing the value takes care of remapping the new value.
ReplaceValueWith(SDValue(N, i), SDValue(M, i));
- // Fall through.
+ assert(N->getNodeId() == NewNode && "Unexpected node state!");
+ // The node continues to live on as part of the NewNode fungus that
+ // grows on top of the useful nodes. Nothing more needs to be done
+ // with it - move on to the next node.
+ continue;
}
if (i == NumOperands) {
- DEBUG(cerr << "Legally typed node: "; N->dump(&DAG); cerr << "\n");
+ DEBUG(dbgs() << "Legally typed node: "; N->dump(&DAG); dbgs() << "\n");
}
}
NodeDone:
// In a debug build, scan all the nodes to make sure we found them all. This
// ensures that there are no cycles and that everything got processed.
#ifndef NDEBUG
- for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
- E = DAG.allnodes_end(); I != E; ++I) {
+ for (SDNode &Node : DAG.allnodes()) {
bool Failed = false;
// Check that all result types are legal.
- if (!IgnoreNodeResults(I))
- for (unsigned i = 0, NumVals = I->getNumValues(); i < NumVals; ++i)
- if (!isTypeLegal(I->getValueType(i))) {
- cerr << "Result type " << i << " illegal!\n";
+ if (!IgnoreNodeResults(&Node))
+ for (unsigned i = 0, NumVals = Node.getNumValues(); i < NumVals; ++i)
+ if (!isTypeLegal(Node.getValueType(i))) {
+ dbgs() << "Result type " << i << " illegal!\n";
Failed = true;
}
// Check that all operand types are legal.
- for (unsigned i = 0, NumOps = I->getNumOperands(); i < NumOps; ++i)
- if (!IgnoreNodeResults(I->getOperand(i).getNode()) &&
- !isTypeLegal(I->getOperand(i).getValueType())) {
- cerr << "Operand type " << i << " illegal!\n";
+ for (unsigned i = 0, NumOps = Node.getNumOperands(); i < NumOps; ++i)
+ if (!IgnoreNodeResults(Node.getOperand(i).getNode()) &&
+ !isTypeLegal(Node.getOperand(i).getValueType())) {
+ dbgs() << "Operand type " << i << " illegal!\n";
Failed = true;
}
- if (I->getNodeId() != Processed) {
- if (I->getNodeId() == NewNode)
- cerr << "New node not analyzed?\n";
- else if (I->getNodeId() == Unanalyzed)
- cerr << "Unanalyzed node not noticed?\n";
- else if (I->getNodeId() > 0)
- cerr << "Operand not processed?\n";
- else if (I->getNodeId() == ReadyToProcess)
- cerr << "Not added to worklist?\n";
+ if (Node.getNodeId() != Processed) {
+ if (Node.getNodeId() == NewNode)
+ dbgs() << "New node not analyzed?\n";
+ else if (Node.getNodeId() == Unanalyzed)
+ dbgs() << "Unanalyzed node not noticed?\n";
+ else if (Node.getNodeId() > 0)
+ dbgs() << "Operand not processed?\n";
+ else if (Node.getNodeId() == ReadyToProcess)
+ dbgs() << "Not added to worklist?\n";
Failed = true;
}
if (Failed) {
- I->dump(&DAG); cerr << "\n";
- abort();
+ Node.dump(&DAG); dbgs() << "\n";
+ llvm_unreachable(nullptr);
}
}
#endif
NewOps.push_back(Op);
} else if (Op != OrigOp) {
// This is the first operand to change - add all operands so far.
- for (unsigned j = 0; j < i; ++j)
- NewOps.push_back(N->getOperand(j));
+ NewOps.append(N->op_begin(), N->op_begin() + i);
NewOps.push_back(Op);
}
}
// Some operands changed - update the node.
if (!NewOps.empty()) {
- SDNode *M = DAG.UpdateNodeOperands(SDValue(N, 0), &NewOps[0],
- NewOps.size()).getNode();
+ SDNode *M = DAG.UpdateNodeOperands(N, NewOps);
if (M != N) {
// The node morphed into a different node. Normally for this to happen
// the original node would have to be marked NewNode. However this can
RemapValue(I->second);
}
+ for (DenseMap<SDValue, SDValue>::iterator I = WidenedVectors.begin(),
+ E = WidenedVectors.end(); I != E; ++I) {
+ assert(I->first.getNode() != N);
+ RemapValue(I->second);
+ }
+
for (DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator
I = ExpandedIntegers.begin(), E = ExpandedIntegers.end(); I != E; ++I){
assert(I->first.getNode() != N);
// replaced with other values.
RemapValue(I->second);
N = I->second;
- assert(N.getNode()->getNodeId() != NewNode && "Mapped to new node!");
+
+ // Note that it is possible to have N.getNode()->getNodeId() == NewNode at
+ // this point because it is possible for a node to be put in the map before
+ // being processed.
}
}
namespace {
/// NodeUpdateListener - This class is a DAGUpdateListener that listens for
/// updates to nodes and recomputes their ready state.
- class VISIBILITY_HIDDEN NodeUpdateListener :
- public SelectionDAG::DAGUpdateListener {
+ class NodeUpdateListener : public SelectionDAG::DAGUpdateListener {
DAGTypeLegalizer &DTL;
- SmallVectorImpl<SDNode*> &NodesToAnalyze;
- SmallPtrSet<SDNode*, 16> &NodesDeleted;
+ SmallSetVector<SDNode*, 16> &NodesToAnalyze;
public:
explicit NodeUpdateListener(DAGTypeLegalizer &dtl,
- SmallVectorImpl<SDNode*> &nta,
- SmallPtrSet<SDNode*, 16> &nd)
- : DTL(dtl), NodesToAnalyze(nta), NodesDeleted(nd) {}
+ SmallSetVector<SDNode*, 16> &nta)
+ : SelectionDAG::DAGUpdateListener(dtl.getDAG()),
+ DTL(dtl), NodesToAnalyze(nta) {}
- virtual void NodeDeleted(SDNode *N, SDNode *E) {
+ void NodeDeleted(SDNode *N, SDNode *E) override {
assert(N->getNodeId() != DAGTypeLegalizer::ReadyToProcess &&
N->getNodeId() != DAGTypeLegalizer::Processed &&
"Invalid node ID for RAUW deletion!");
DTL.NoteDeletion(N, E);
// In theory the deleted node could also have been scheduled for analysis.
- // So add it to the set of nodes which will not be analyzed.
- NodesDeleted.insert(N);
+ // So remove it from the set of nodes which will be analyzed.
+ NodesToAnalyze.remove(N);
// In general nothing needs to be done for E, since it didn't change but
// only gained new uses. However N -> E was just added to ReplacedValues,
// and the result of a ReplacedValues mapping is not allowed to be marked
// NewNode. So if E is marked NewNode, then it needs to be analyzed.
if (E->getNodeId() == DAGTypeLegalizer::NewNode)
- NodesToAnalyze.push_back(E);
+ NodesToAnalyze.insert(E);
}
- virtual void NodeUpdated(SDNode *N) {
+ void NodeUpdated(SDNode *N) override {
// Node updates can mean pretty much anything. It is possible that an
// operand was set to something already processed (f.e.) in which case
// this node could become ready. Recompute its flags.
assert(N->getNodeId() != DAGTypeLegalizer::ReadyToProcess &&
N->getNodeId() != DAGTypeLegalizer::Processed &&
"Invalid node ID for RAUW deletion!");
- NodesToAnalyze.push_back(N);
+ N->setNodeId(DAGTypeLegalizer::NewNode);
+ NodesToAnalyze.insert(N);
}
};
}
/// value. Update the DAG and NodeIds replacing any uses of From to use To
/// instead.
void DAGTypeLegalizer::ReplaceValueWith(SDValue From, SDValue To) {
- assert(From.getNode()->getNodeId() == ReadyToProcess &&
- "Only the node being processed may be remapped!");
assert(From.getNode() != To.getNode() && "Potential legalization loop!");
// If expansion produced new nodes, make sure they are properly marked.
// Anything that used the old node should now use the new one. Note that this
// can potentially cause recursive merging.
- SmallVector<SDNode*, 16> NodesToAnalyze;
- SmallPtrSet<SDNode*, 16> NodesDeleted;
- NodeUpdateListener NUL(*this, NodesToAnalyze, NodesDeleted);
- DAG.ReplaceAllUsesOfValueWith(From, To, &NUL);
-
- // The old node may still be present in a map like ExpandedIntegers or
- // PromotedIntegers. Inform maps about the replacement.
- ReplacedValues[From] = To;
-
- // Process the list of nodes that need to be reanalyzed.
- while (!NodesToAnalyze.empty()) {
- SDNode *N = NodesToAnalyze.back();
- NodesToAnalyze.pop_back();
-
- // Do not analyze deleted nodes!
- if (NodesDeleted.count(N))
- continue;
+ SmallSetVector<SDNode*, 16> NodesToAnalyze;
+ NodeUpdateListener NUL(*this, NodesToAnalyze);
+ do {
+ DAG.ReplaceAllUsesOfValueWith(From, To);
+
+ // The old node may still be present in a map like ExpandedIntegers or
+ // PromotedIntegers. Inform maps about the replacement.
+ ReplacedValues[From] = To;
+
+ // Process the list of nodes that need to be reanalyzed.
+ while (!NodesToAnalyze.empty()) {
+ SDNode *N = NodesToAnalyze.back();
+ NodesToAnalyze.pop_back();
+ if (N->getNodeId() != DAGTypeLegalizer::NewNode)
+ // The node was analyzed while reanalyzing an earlier node - it is safe
+ // to skip. Note that this is not a morphing node - otherwise it would
+ // still be marked NewNode.
+ continue;
- // Analyze the node's operands and recalculate the node ID.
- assert(N->getNodeId() != DAGTypeLegalizer::ReadyToProcess &&
- N->getNodeId() != DAGTypeLegalizer::Processed &&
- "Invalid node ID for RAUW analysis!");
- N->setNodeId(NewNode);
- SDNode *M = AnalyzeNewNode(N);
- if (M != N) {
- // The node morphed into a different node. Make everyone use the new node
- // instead.
- assert(M->getNodeId() != NewNode && "Analysis resulted in NewNode!");
- assert(N->getNumValues() == M->getNumValues() &&
- "Node morphing changed the number of results!");
- for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
- SDValue OldVal(N, i);
- SDValue NewVal(M, i);
- if (M->getNodeId() == Processed)
- RemapValue(NewVal);
- DAG.ReplaceAllUsesOfValueWith(OldVal, NewVal, &NUL);
+ // Analyze the node's operands and recalculate the node ID.
+ SDNode *M = AnalyzeNewNode(N);
+ if (M != N) {
+ // The node morphed into a different node. Make everyone use the new
+ // node instead.
+ assert(M->getNodeId() != NewNode && "Analysis resulted in NewNode!");
+ assert(N->getNumValues() == M->getNumValues() &&
+ "Node morphing changed the number of results!");
+ for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
+ SDValue OldVal(N, i);
+ SDValue NewVal(M, i);
+ if (M->getNodeId() == Processed)
+ RemapValue(NewVal);
+ DAG.ReplaceAllUsesOfValueWith(OldVal, NewVal);
+ // OldVal may be a target of the ReplacedValues map which was marked
+ // NewNode to force reanalysis because it was updated. Ensure that
+ // anything that ReplacedValues mapped to OldVal will now be mapped
+ // all the way to NewVal.
+ ReplacedValues[OldVal] = NewVal;
+ }
+ // The original node continues to exist in the DAG, marked NewNode.
}
- // The original node continues to exist in the DAG, marked NewNode.
}
- }
+ // When recursively update nodes with new nodes, it is possible to have
+ // new uses of From due to CSE. If this happens, replace the new uses of
+ // From with To.
+ } while (!From.use_empty());
}
void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) {
+ assert(Result.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+ "Invalid type for promoted integer");
AnalyzeNewValue(Result);
SDValue &OpEntry = PromotedIntegers[Op];
- assert(OpEntry.getNode() == 0 && "Node is already promoted!");
+ assert(!OpEntry.getNode() && "Node is already promoted!");
OpEntry = Result;
}
void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) {
+ assert(Result.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+ "Invalid type for softened float");
AnalyzeNewValue(Result);
SDValue &OpEntry = SoftenedFloats[Op];
- assert(OpEntry.getNode() == 0 && "Node is already converted to integer!");
+ assert(!OpEntry.getNode() && "Node is already converted to integer!");
+ OpEntry = Result;
+}
+
+void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) {
+ assert(Result.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+ "Invalid type for promoted float");
+ AnalyzeNewValue(Result);
+
+ SDValue &OpEntry = PromotedFloats[Op];
+ assert(!OpEntry.getNode() && "Node is already promoted!");
OpEntry = Result;
}
void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) {
+ // Note that in some cases vector operation operands may be greater than
+ // the vector element type. For example BUILD_VECTOR of type <1 x i1> with
+ // a constant i8 operand.
+ assert(Result.getValueType().getSizeInBits() >=
+ Op.getValueType().getVectorElementType().getSizeInBits() &&
+ "Invalid type for scalarized vector");
AnalyzeNewValue(Result);
SDValue &OpEntry = ScalarizedVectors[Op];
- assert(OpEntry.getNode() == 0 && "Node is already scalarized!");
+ assert(!OpEntry.getNode() && "Node is already scalarized!");
OpEntry = Result;
}
void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo,
SDValue Hi) {
+ assert(Lo.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+ Hi.getValueType() == Lo.getValueType() &&
+ "Invalid type for expanded integer");
// Lo/Hi may have been newly allocated, if so, add nodeid's as relevant.
AnalyzeNewValue(Lo);
AnalyzeNewValue(Hi);
// Remember that this is the result of the node.
std::pair<SDValue, SDValue> &Entry = ExpandedIntegers[Op];
- assert(Entry.first.getNode() == 0 && "Node already expanded");
+ assert(!Entry.first.getNode() && "Node already expanded");
Entry.first = Lo;
Entry.second = Hi;
}
void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo,
SDValue Hi) {
+ assert(Lo.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+ Hi.getValueType() == Lo.getValueType() &&
+ "Invalid type for expanded float");
// Lo/Hi may have been newly allocated, if so, add nodeid's as relevant.
AnalyzeNewValue(Lo);
AnalyzeNewValue(Hi);
// Remember that this is the result of the node.
std::pair<SDValue, SDValue> &Entry = ExpandedFloats[Op];
- assert(Entry.first.getNode() == 0 && "Node already expanded");
+ assert(!Entry.first.getNode() && "Node already expanded");
Entry.first = Lo;
Entry.second = Hi;
}
void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo,
SDValue Hi) {
+ assert(Lo.getValueType().getVectorElementType() ==
+ Op.getValueType().getVectorElementType() &&
+ 2*Lo.getValueType().getVectorNumElements() ==
+ Op.getValueType().getVectorNumElements() &&
+ Hi.getValueType() == Lo.getValueType() &&
+ "Invalid type for split vector");
// Lo/Hi may have been newly allocated, if so, add nodeid's as relevant.
AnalyzeNewValue(Lo);
AnalyzeNewValue(Hi);
// Remember that this is the result of the node.
std::pair<SDValue, SDValue> &Entry = SplitVectors[Op];
- assert(Entry.first.getNode() == 0 && "Node already split");
+ assert(!Entry.first.getNode() && "Node already split");
Entry.first = Lo;
Entry.second = Hi;
}
void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) {
+ assert(Result.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+ "Invalid type for widened vector");
AnalyzeNewValue(Result);
SDValue &OpEntry = WidenedVectors[Op];
- assert(OpEntry.getNode() == 0 && "Node is already promoted!");
+ assert(!OpEntry.getNode() && "Node already widened!");
OpEntry = Result;
}
-// Set to ignore result
-void DAGTypeLegalizer::SetIgnoredNodeResult(SDNode* N) {
- IgnoredNodesResultsSet.insert(N);
-}
//===----------------------------------------------------------------------===//
// Utilities.
/// BitConvertToInteger - Convert to an integer of the same size.
SDValue DAGTypeLegalizer::BitConvertToInteger(SDValue Op) {
unsigned BitWidth = Op.getValueType().getSizeInBits();
- return DAG.getNode(ISD::BIT_CONVERT, MVT::getIntegerVT(BitWidth), Op);
+ return DAG.getNode(ISD::BITCAST, SDLoc(Op),
+ EVT::getIntegerVT(*DAG.getContext(), BitWidth), Op);
+}
+
+/// BitConvertVectorToIntegerVector - Convert to a vector of integers of the
+/// same size.
+SDValue DAGTypeLegalizer::BitConvertVectorToIntegerVector(SDValue Op) {
+ assert(Op.getValueType().isVector() && "Only applies to vectors!");
+ unsigned EltWidth = Op.getValueType().getVectorElementType().getSizeInBits();
+ EVT EltNVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth);
+ unsigned NumElts = Op.getValueType().getVectorNumElements();
+ return DAG.getNode(ISD::BITCAST, SDLoc(Op),
+ EVT::getVectorVT(*DAG.getContext(), EltNVT, NumElts), Op);
}
SDValue DAGTypeLegalizer::CreateStackStoreLoad(SDValue Op,
- MVT DestVT) {
+ EVT DestVT) {
+ SDLoc dl(Op);
// Create the stack frame object. Make sure it is aligned for both
// the source and destination types.
SDValue StackPtr = DAG.CreateStackTemporary(Op.getValueType(), DestVT);
// Emit a store to the stack slot.
- SDValue Store = DAG.getStore(DAG.getEntryNode(), Op, StackPtr, NULL, 0);
+ SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op, StackPtr,
+ MachinePointerInfo(), false, false, 0);
// Result is a load from the stack slot.
- return DAG.getLoad(DestVT, Store, StackPtr, NULL, 0);
+ return DAG.getLoad(DestVT, dl, Store, StackPtr, MachinePointerInfo(),
+ false, false, false, 0);
}
-/// CustomLowerResults - Replace the node's results with custom code provided
+/// CustomLowerNode - Replace the node's results with custom code provided
/// by the target and return "true", or do nothing and return "false".
-bool DAGTypeLegalizer::CustomLowerResults(SDNode *N, unsigned ResNo) {
+/// The last parameter is FALSE if we are dealing with a node with legal
+/// result types and illegal operand. The second parameter denotes the type of
+/// illegal OperandNo in that case.
+/// The last parameter being TRUE means we are dealing with a
+/// node with illegal result types. The second parameter denotes the type of
+/// illegal ResNo in that case.
+bool DAGTypeLegalizer::CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult) {
// See if the target wants to custom lower this node.
- if (TLI.getOperationAction(N->getOpcode(), N->getValueType(ResNo)) !=
- TargetLowering::Custom)
+ if (TLI.getOperationAction(N->getOpcode(), VT) != TargetLowering::Custom)
return false;
SmallVector<SDValue, 8> Results;
- TLI.ReplaceNodeResults(N, Results, DAG);
+ if (LegalizeResult)
+ TLI.ReplaceNodeResults(N, Results, DAG);
+ else
+ TLI.LowerOperationWrapper(N, Results, DAG);
+
if (Results.empty())
// The target didn't want to custom lower it after all.
return false;
+ // When called from DAGTypeLegalizer::ExpandIntegerResult, we might need to
+ // provide the same kind of custom splitting behavior.
+ if (Results.size() == N->getNumValues() + 1 && LegalizeResult) {
+ // We've legalized a return type by splitting it. If there is a chain,
+ // replace that too.
+ SetExpandedInteger(SDValue(N, 0), Results[0], Results[1]);
+ if (N->getNumValues() > 1)
+ ReplaceValueWith(SDValue(N, 1), Results[2]);
+ return true;
+ }
+
// Make everything that once used N's values now use those in Results instead.
assert(Results.size() == N->getNumValues() &&
"Custom lowering returned the wrong number of results!");
- for (unsigned i = 0, e = Results.size(); i != e; ++i)
+ for (unsigned i = 0, e = Results.size(); i != e; ++i) {
ReplaceValueWith(SDValue(N, i), Results[i]);
+ }
return true;
}
-/// JoinIntegers - Build an integer with low bits Lo and high bits Hi.
-SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
- MVT LVT = Lo.getValueType();
- MVT HVT = Hi.getValueType();
- MVT NVT = MVT::getIntegerVT(LVT.getSizeInBits() + HVT.getSizeInBits());
- Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, Lo);
- Hi = DAG.getNode(ISD::ANY_EXTEND, NVT, Hi);
- Hi = DAG.getNode(ISD::SHL, NVT, Hi, DAG.getConstant(LVT.getSizeInBits(),
- TLI.getShiftAmountTy()));
- return DAG.getNode(ISD::OR, NVT, Lo, Hi);
+/// CustomWidenLowerNode - Widen the node's results with custom code provided
+/// by the target and return "true", or do nothing and return "false".
+bool DAGTypeLegalizer::CustomWidenLowerNode(SDNode *N, EVT VT) {
+ // See if the target wants to custom lower this node.
+ if (TLI.getOperationAction(N->getOpcode(), VT) != TargetLowering::Custom)
+ return false;
+
+ SmallVector<SDValue, 8> Results;
+ TLI.ReplaceNodeResults(N, Results, DAG);
+
+ if (Results.empty())
+ // The target didn't want to custom widen lower its result after all.
+ return false;
+
+ // Update the widening map.
+ assert(Results.size() == N->getNumValues() &&
+ "Custom lowering returned the wrong number of results!");
+ for (unsigned i = 0, e = Results.size(); i != e; ++i)
+ SetWidenedVector(SDValue(N, i), Results[i]);
+ return true;
}
-/// SplitInteger - Return the lower LoVT bits of Op in Lo and the upper HiVT
-/// bits in Hi.
-void DAGTypeLegalizer::SplitInteger(SDValue Op,
- MVT LoVT, MVT HiVT,
- SDValue &Lo, SDValue &Hi) {
- assert(LoVT.getSizeInBits() + HiVT.getSizeInBits() ==
- Op.getValueType().getSizeInBits() && "Invalid integer splitting!");
- Lo = DAG.getNode(ISD::TRUNCATE, LoVT, Op);
- Hi = DAG.getNode(ISD::SRL, Op.getValueType(), Op,
- DAG.getConstant(LoVT.getSizeInBits(),
- TLI.getShiftAmountTy()));
- Hi = DAG.getNode(ISD::TRUNCATE, HiVT, Hi);
+SDValue DAGTypeLegalizer::DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo) {
+ for (unsigned i = 0, e = N->getNumValues(); i != e; ++i)
+ if (i != ResNo)
+ ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i)));
+ return SDValue(N->getOperand(ResNo));
}
-/// SplitInteger - Return the lower and upper halves of Op's bits in a value
-/// type half the size of Op's.
-void DAGTypeLegalizer::SplitInteger(SDValue Op,
- SDValue &Lo, SDValue &Hi) {
- MVT HalfVT = MVT::getIntegerVT(Op.getValueType().getSizeInBits()/2);
- SplitInteger(Op, HalfVT, HalfVT, Lo, Hi);
+/// GetPairElements - Use ISD::EXTRACT_ELEMENT nodes to extract the low and
+/// high parts of the given value.
+void DAGTypeLegalizer::GetPairElements(SDValue Pair,
+ SDValue &Lo, SDValue &Hi) {
+ SDLoc dl(Pair);
+ EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Pair.getValueType());
+ Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
+ DAG.getIntPtrConstant(0, dl));
+ Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
+ DAG.getIntPtrConstant(1, dl));
}
-/// MakeLibCall - Generate a libcall taking the given operands as arguments and
-/// returning a result of type RetVT.
-SDValue DAGTypeLegalizer::MakeLibCall(RTLIB::Libcall LC, MVT RetVT,
- const SDValue *Ops, unsigned NumOps,
- bool isSigned) {
- TargetLowering::ArgListTy Args;
- Args.reserve(NumOps);
+SDValue DAGTypeLegalizer::GetVectorElementPointer(SDValue VecPtr, EVT EltVT,
+ SDValue Index) {
+ SDLoc dl(Index);
+ // Make sure the index type is big enough to compute in.
+ Index = DAG.getZExtOrTrunc(Index, dl, TLI.getPointerTy(DAG.getDataLayout()));
- TargetLowering::ArgListEntry Entry;
- for (unsigned i = 0; i != NumOps; ++i) {
- Entry.Node = Ops[i];
- Entry.Ty = Entry.Node.getValueType().getTypeForMVT();
- Entry.isSExt = isSigned;
- Entry.isZExt = !isSigned;
- Args.push_back(Entry);
- }
- SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
- TLI.getPointerTy());
+ // Calculate the element offset and add it to the pointer.
+ unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
- const Type *RetTy = RetVT.getTypeForMVT();
- std::pair<SDValue,SDValue> CallInfo =
- TLI.LowerCallTo(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false,
- false, CallingConv::C, false, Callee, Args, DAG);
- return CallInfo.first;
+ Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
+ DAG.getConstant(EltSize, dl, Index.getValueType()));
+ return DAG.getNode(ISD::ADD, dl, Index.getValueType(), Index, VecPtr);
+}
+
+/// JoinIntegers - Build an integer with low bits Lo and high bits Hi.
+SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
+ // Arbitrarily use dlHi for result SDLoc
+ SDLoc dlHi(Hi);
+ SDLoc dlLo(Lo);
+ EVT LVT = Lo.getValueType();
+ EVT HVT = Hi.getValueType();
+ EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
+ LVT.getSizeInBits() + HVT.getSizeInBits());
+
+ Lo = DAG.getNode(ISD::ZERO_EXTEND, dlLo, NVT, Lo);
+ Hi = DAG.getNode(ISD::ANY_EXTEND, dlHi, NVT, Hi);
+ Hi = DAG.getNode(ISD::SHL, dlHi, NVT, Hi,
+ DAG.getConstant(LVT.getSizeInBits(), dlHi,
+ TLI.getPointerTy(DAG.getDataLayout())));
+ return DAG.getNode(ISD::OR, dlHi, NVT, Lo, Hi);
}
/// LibCallify - Convert the node into a libcall with the same prototype.
SDValue DAGTypeLegalizer::LibCallify(RTLIB::Libcall LC, SDNode *N,
bool isSigned) {
unsigned NumOps = N->getNumOperands();
+ SDLoc dl(N);
if (NumOps == 0) {
- return MakeLibCall(LC, N->getValueType(0), 0, 0, isSigned);
+ return TLI.makeLibCall(DAG, LC, N->getValueType(0), nullptr, 0, isSigned,
+ dl).first;
} else if (NumOps == 1) {
SDValue Op = N->getOperand(0);
- return MakeLibCall(LC, N->getValueType(0), &Op, 1, isSigned);
+ return TLI.makeLibCall(DAG, LC, N->getValueType(0), &Op, 1, isSigned,
+ dl).first;
} else if (NumOps == 2) {
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
- return MakeLibCall(LC, N->getValueType(0), Ops, 2, isSigned);
+ return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, 2, isSigned,
+ dl).first;
}
SmallVector<SDValue, 8> Ops(NumOps);
for (unsigned i = 0; i < NumOps; ++i)
Ops[i] = N->getOperand(i);
- return MakeLibCall(LC, N->getValueType(0), &Ops[0], NumOps, isSigned);
+ return TLI.makeLibCall(DAG, LC, N->getValueType(0),
+ &Ops[0], NumOps, isSigned, dl).first;
}
-SDValue DAGTypeLegalizer::GetVectorElementPointer(SDValue VecPtr, MVT EltVT,
- SDValue Index) {
- // Make sure the index type is big enough to compute in.
- if (Index.getValueType().bitsGT(TLI.getPointerTy()))
- Index = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), Index);
- else
- Index = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), Index);
+// ExpandChainLibCall - Expand a node into a call to a libcall. Similar to
+// ExpandLibCall except that the first operand is the in-chain.
+std::pair<SDValue, SDValue>
+DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC,
+ SDNode *Node,
+ bool isSigned) {
+ SDValue InChain = Node->getOperand(0);
- // Calculate the element offset and add it to the pointer.
- unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
-
- Index = DAG.getNode(ISD::MUL, Index.getValueType(), Index,
- DAG.getConstant(EltSize, Index.getValueType()));
- return DAG.getNode(ISD::ADD, Index.getValueType(), Index, VecPtr);
-}
-
-/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
-/// which is split into two not necessarily identical pieces.
-void DAGTypeLegalizer::GetSplitDestVTs(MVT InVT, MVT &LoVT, MVT &HiVT) {
- if (!InVT.isVector()) {
- LoVT = HiVT = TLI.getTypeToTransformTo(InVT);
- } else {
- MVT NewEltVT = InVT.getVectorElementType();
- unsigned NumElements = InVT.getVectorNumElements();
- if ((NumElements & (NumElements-1)) == 0) { // Simple power of two vector.
- NumElements >>= 1;
- LoVT = HiVT = MVT::getVectorVT(NewEltVT, NumElements);
- } else { // Non-power-of-two vectors.
- unsigned NewNumElts_Lo = 1 << Log2_32(NumElements);
- unsigned NewNumElts_Hi = NumElements - NewNumElts_Lo;
- LoVT = MVT::getVectorVT(NewEltVT, NewNumElts_Lo);
- HiVT = MVT::getVectorVT(NewEltVT, NewNumElts_Hi);
- }
+ TargetLowering::ArgListTy Args;
+ TargetLowering::ArgListEntry Entry;
+ for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
+ EVT ArgVT = Node->getOperand(i).getValueType();
+ Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+ Entry.Node = Node->getOperand(i);
+ Entry.Ty = ArgTy;
+ Entry.isSExt = isSigned;
+ Entry.isZExt = !isSigned;
+ Args.push_back(Entry);
}
+ SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
+ TLI.getPointerTy(DAG.getDataLayout()));
+
+ Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
+
+ TargetLowering::CallLoweringInfo CLI(DAG);
+ CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
+ .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0)
+ .setSExtResult(isSigned).setZExtResult(!isSigned);
+
+ std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
+
+ return CallInfo;
+}
+
+/// PromoteTargetBoolean - Promote the given target boolean to a target boolean
+/// of the given type. A target boolean is an integer value, not necessarily of
+/// type i1, the bits of which conform to getBooleanContents.
+///
+/// ValVT is the type of values that produced the boolean.
+SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT ValVT) {
+ SDLoc dl(Bool);
+ EVT BoolVT = getSetCCResultType(ValVT);
+ ISD::NodeType ExtendCode =
+ TargetLowering::getExtendForContent(TLI.getBooleanContents(ValVT));
+ return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
+}
+
+/// SplitInteger - Return the lower LoVT bits of Op in Lo and the upper HiVT
+/// bits in Hi.
+void DAGTypeLegalizer::SplitInteger(SDValue Op,
+ EVT LoVT, EVT HiVT,
+ SDValue &Lo, SDValue &Hi) {
+ SDLoc dl(Op);
+ assert(LoVT.getSizeInBits() + HiVT.getSizeInBits() ==
+ Op.getValueType().getSizeInBits() && "Invalid integer splitting!");
+ Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Op);
+ Hi = DAG.getNode(ISD::SRL, dl, Op.getValueType(), Op,
+ DAG.getConstant(LoVT.getSizeInBits(), dl,
+ TLI.getPointerTy(DAG.getDataLayout())));
+ Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
+}
+
+/// SplitInteger - Return the lower and upper halves of Op's bits in a value
+/// type half the size of Op's.
+void DAGTypeLegalizer::SplitInteger(SDValue Op,
+ SDValue &Lo, SDValue &Hi) {
+ EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(),
+ Op.getValueType().getSizeInBits()/2);
+ SplitInteger(Op, HalfVT, HalfVT, Lo, Hi);
}
projects / oota-llvm.git / blobdiff