#include "llvm/CodeGen/SelectionDAG.h"
#include "SDNodeDbgValue.h"
+#include "llvm/ADT/APSInt.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/Target/TargetSubtargetInfo.h"
#include <algorithm>
#include <cmath>
+#include <utility>
using namespace llvm;
if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
bool IsAllUndef = true;
- for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
- if (N->getOperand(i).getOpcode() == ISD::UNDEF)
+ for (const SDValue &Op : N->op_values()) {
+ if (Op.getOpcode() == ISD::UNDEF)
continue;
IsAllUndef = false;
// Do not accept build_vectors that aren't all constants or which have non-0
// We only want to check enough bits to cover the vector elements, because
// we care if the resultant vector is all zeros, not whether the individual
// constants are.
- SDValue Zero = N->getOperand(i);
unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
- if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
+ if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
if (CN->getAPIntValue().countTrailingZeros() < EltSize)
return false;
- } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
+ } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
return false;
} else
if (N->getOpcode() != ISD::BUILD_VECTOR)
return false;
- for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
- SDValue Op = N->getOperand(i);
+ for (const SDValue &Op : N->op_values()) {
if (Op.getOpcode() == ISD::UNDEF)
continue;
if (!isa<ConstantSDNode>(Op))
return true;
}
-/// isScalarToVector - Return true if the specified node is a
-/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
-/// element is not an undef.
-bool ISD::isScalarToVector(const SDNode *N) {
- if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
- return true;
-
+/// \brief Return true if the specified node is a BUILD_VECTOR node of
+/// all ConstantFPSDNode or undef.
+bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
if (N->getOpcode() != ISD::BUILD_VECTOR)
return false;
- if (N->getOperand(0).getOpcode() == ISD::UNDEF)
- return false;
- unsigned NumElems = N->getNumOperands();
- if (NumElems == 1)
- return false;
- for (unsigned i = 1; i < NumElems; ++i) {
- SDValue V = N->getOperand(i);
- if (V.getOpcode() != ISD::UNDEF)
+
+ for (const SDValue &Op : N->op_values()) {
+ if (Op.getOpcode() == ISD::UNDEF)
+ continue;
+ if (!isa<ConstantFPSDNode>(Op))
return false;
}
return true;
if (N->getNumOperands() == 0)
return false;
- for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
- if (N->getOperand(i).getOpcode() != ISD::UNDEF)
+ for (const SDValue &Op : N->op_values())
+ if (Op.getOpcode() != ISD::UNDEF)
return false;
return true;
}
}
-static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
- bool exact) {
- ID.AddBoolean(nuw);
- ID.AddBoolean(nsw);
- ID.AddBoolean(exact);
-}
-
-/// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
-static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
- bool nuw, bool nsw, bool exact) {
- if (isBinOpWithFlags(Opcode))
- AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
-}
-
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
SDVTList VTList, ArrayRef<SDValue> OpList) {
AddNodeIDOpcode(ID, OpC);
AddNodeIDOperands(ID, OpList);
}
-/// AddNodeIDCustom - If this is an SDNode with special info, add this info to
-/// the NodeID data.
+/// If this is an SDNode with special info, add this info to the NodeID data.
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
switch (N->getOpcode()) {
case ISD::TargetExternalSymbol:
case ISD::ExternalSymbol:
+ case ISD::MCSymbol:
llvm_unreachable("Should only be used on nodes with operands");
default: break; // Normal nodes don't need extra info.
case ISD::TargetConstant:
ID.AddInteger(ST->getPointerInfo().getAddrSpace());
break;
}
- case ISD::SDIV:
- case ISD::UDIV:
- case ISD::SRA:
- case ISD::SRL:
- case ISD::MUL:
- case ISD::ADD:
- case ISD::SUB:
- case ISD::SHL: {
- const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
- AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
- BinNode->hasNoSignedWrap(), BinNode->isExact());
- break;
- }
case ISD::ATOMIC_CMP_SWAP:
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
case ISD::ATOMIC_SWAP:
SmallVector<SDNode*, 128> DeadNodes;
// Add all obviously-dead nodes to the DeadNodes worklist.
- for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
- if (I->use_empty())
- DeadNodes.push_back(I);
+ for (SDNode &Node : allnodes())
+ if (Node.use_empty())
+ DeadNodes.push_back(&Node);
RemoveDeadNodes(DeadNodes);
void SelectionDAG::InsertNode(SDNode *N) {
AllNodes.push_back(N);
#ifndef NDEBUG
+ N->PersistentId = NextPersistentId++;
VerifySDNode(N);
#endif
}
ESN->getTargetFlags()));
break;
}
+ case ISD::MCSymbol: {
+ auto *MCSN = cast<MCSymbolSDNode>(N);
+ Erased = MCSymbols.erase(MCSN->getMCSymbol());
+ break;
+ }
case ISD::VALUETYPE: {
EVT VT = cast<VTSDNode>(N)->getVT();
if (VT.isExtended()) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
AddNodeIDCustom(ID, N);
- SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+ SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
+ if (Node)
+ if (const SDNodeFlags *Flags = N->getFlags())
+ Node->intersectFlagsWith(Flags);
return Node;
}
FoldingSetNodeID ID;
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
AddNodeIDCustom(ID, N);
- SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+ SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
+ if (Node)
+ if (const SDNodeFlags *Flags = N->getFlags())
+ Node->intersectFlagsWith(Flags);
return Node;
}
FoldingSetNodeID ID;
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
AddNodeIDCustom(ID, N);
- SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+ SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
+ if (Node)
+ if (const SDNodeFlags *Flags = N->getFlags())
+ Node->intersectFlagsWith(Flags);
return Node;
}
PointerType::get(Type::getInt8Ty(*getContext()), 0) :
VT.getTypeForEVT(*getContext());
- return TLI->getDataLayout()->getABITypeAlignment(Ty);
+ return getDataLayout().getABITypeAlignment(Ty);
}
// EntryNode could meaningfully have debug info if we can find it...
EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
UpdateListeners(nullptr) {
- AllNodes.push_back(&EntryNode);
+ InsertNode(&EntryNode);
DbgInfo = new SDDbgInfo();
}
assert(&*AllNodes.begin() == &EntryNode);
AllNodes.remove(AllNodes.begin());
while (!AllNodes.empty())
- DeallocateNode(AllNodes.begin());
+ DeallocateNode(&AllNodes.front());
+#ifndef NDEBUG
+ NextPersistentId = 0;
+#endif
}
BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
SDVTList VTs, SDValue N1,
- SDValue N2, bool nuw, bool nsw,
- bool exact) {
+ SDValue N2,
+ const SDNodeFlags *Flags) {
if (isBinOpWithFlags(Opcode)) {
+ // If no flags were passed in, use a default flags object.
+ SDNodeFlags F;
+ if (Flags == nullptr)
+ Flags = &F;
+
BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
- Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
- FN->setHasNoUnsignedWrap(nuw);
- FN->setHasNoSignedWrap(nsw);
- FN->setIsExact(exact);
+ Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags);
return FN;
}
return N;
}
+SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
+ void *&InsertPos) {
+ SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+ if (N) {
+ switch (N->getOpcode()) {
+ default: break;
+ case ISD::Constant:
+ case ISD::ConstantFP:
+ llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
+ "debug location. Use another overload.");
+ }
+ }
+ return N;
+}
+
+SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
+ DebugLoc DL, void *&InsertPos) {
+ SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+ if (N) {
+ switch (N->getOpcode()) {
+ default: break; // Process only regular (non-target) constant nodes.
+ case ISD::Constant:
+ case ISD::ConstantFP:
+ // Erase debug location from the node if the node is used at several
+ // different places to do not propagate one location to all uses as it
+ // leads to incorrect debug info.
+ if (N->getDebugLoc() != DL)
+ N->setDebugLoc(DebugLoc());
+ break;
+ }
+ }
+ return N;
+}
+
void SelectionDAG::clear() {
allnodes_clear();
OperandAllocator.Reset();
ExtendedValueTypeNodes.clear();
ExternalSymbols.clear();
TargetExternalSymbols.clear();
+ MCSymbols.clear();
std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
static_cast<CondCodeSDNode*>(nullptr));
std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
static_cast<SDNode*>(nullptr));
EntryNode.UseList = nullptr;
- AllNodes.push_back(&EntryNode);
+ InsertNode(&EntryNode);
Root = getEntryNode();
DbgInfo->clear();
}
APInt Imm = APInt::getLowBitsSet(BitWidth,
VT.getSizeInBits());
return getNode(ISD::AND, DL, Op.getValueType(), Op,
- getConstant(Imm, Op.getValueType()));
+ getConstant(Imm, DL, Op.getValueType()));
}
SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
EVT EltVT = VT.getScalarType();
SDValue NegOne =
- getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
+ getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
return getNode(ISD::XOR, DL, VT, Val, NegOne);
}
switch (TLI->getBooleanContents(VT)) {
case TargetLowering::ZeroOrOneBooleanContent:
case TargetLowering::UndefinedBooleanContent:
- TrueValue = getConstant(1, VT);
+ TrueValue = getConstant(1, DL, VT);
break;
case TargetLowering::ZeroOrNegativeOneBooleanContent:
- TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
+ TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL,
VT);
break;
}
return getNode(ISD::XOR, DL, VT, Val, TrueValue);
}
-SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
+SDValue SelectionDAG::getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isT,
+ bool isO) {
EVT EltVT = VT.getScalarType();
assert((EltVT.getSizeInBits() >= 64 ||
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
"getConstant with a uint64_t value that doesn't fit in the type!");
- return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
+ return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
}
-SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
+SDValue SelectionDAG::getConstant(const APInt &Val, SDLoc DL, EVT VT, bool isT,
+ bool isO)
{
- return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
+ return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
}
-SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
- bool isO) {
+SDValue SelectionDAG::getConstant(const ConstantInt &Val, SDLoc DL, EVT VT,
+ bool isT, bool isO) {
assert(VT.isInteger() && "Cannot create FP integer constant!");
EVT EltVT = VT.getScalarType();
SmallVector<SDValue, 2> EltParts;
for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
- .trunc(ViaEltSizeInBits),
+ .trunc(ViaEltSizeInBits), DL,
ViaEltVT, isT, isO));
}
// EltParts is currently in little endian order. If we actually want
// big-endian order then reverse it now.
- if (TLI->isBigEndian())
+ if (getDataLayout().isBigEndian())
std::reverse(EltParts.begin(), EltParts.end());
// The elements must be reversed when the element order is different
ID.AddBoolean(isO);
void *IP = nullptr;
SDNode *N = nullptr;
- if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
+ if ((N = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)))
if (!VT.isVector())
return SDValue(N, 0);
if (!N) {
- N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
+ N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, DL.getDebugLoc(),
+ EltVT);
CSEMap.InsertNode(N, IP);
InsertNode(N);
}
return Result;
}
-SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
- return getConstant(Val, TLI->getPointerTy(), isTarget);
+SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget) {
+ return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
}
-
-SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
- return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
+SDValue SelectionDAG::getConstantFP(const APFloat& V, SDLoc DL, EVT VT,
+ bool isTarget) {
+ return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
}
-SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
+SDValue SelectionDAG::getConstantFP(const ConstantFP& V, SDLoc DL, EVT VT,
+ bool isTarget){
assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
EVT EltVT = VT.getScalarType();
ID.AddPointer(&V);
void *IP = nullptr;
SDNode *N = nullptr;
- if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
+ if ((N = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)))
if (!VT.isVector())
return SDValue(N, 0);
if (!N) {
- N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
+ N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, DL.getDebugLoc(),
+ EltVT);
CSEMap.InsertNode(N, IP);
InsertNode(N);
}
if (VT.isVector()) {
SmallVector<SDValue, 8> Ops;
Ops.assign(VT.getVectorNumElements(), Result);
- // FIXME SDLoc info might be appropriate here
Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
}
return Result;
}
-SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
+SDValue SelectionDAG::getConstantFP(double Val, SDLoc DL, EVT VT,
+ bool isTarget) {
EVT EltVT = VT.getScalarType();
if (EltVT==MVT::f32)
- return getConstantFP(APFloat((float)Val), VT, isTarget);
+ return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
else if (EltVT==MVT::f64)
- return getConstantFP(APFloat(Val), VT, isTarget);
+ return getConstantFP(APFloat(Val), DL, VT, isTarget);
else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
EltVT==MVT::f16) {
bool ignored;
APFloat apf = APFloat(Val);
apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
&ignored);
- return getConstantFP(apf, VT, isTarget);
+ return getConstantFP(apf, DL, VT, isTarget);
} else
llvm_unreachable("Unsupported type in getConstantFP");
}
"Cannot set target flags on target-independent globals");
// Truncate (with sign-extension) the offset value to the pointer size.
- unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
+ unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
if (BitWidth < 64)
Offset = SignExtend64(Offset, BitWidth);
ID.AddInteger(TargetFlags);
ID.AddInteger(GV->getType()->getAddressSpace());
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
AddNodeIDNode(ID, Opc, getVTList(VT), None);
ID.AddInteger(FI);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
ID.AddInteger(JTI);
ID.AddInteger(TargetFlags);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
assert((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals");
if (Alignment == 0)
- Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
+ Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opc, getVTList(VT), None);
ID.AddPointer(C);
ID.AddInteger(TargetFlags);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
assert((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals");
if (Alignment == 0)
- Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
+ Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opc, getVTList(VT), None);
C->addSelectionDAGCSEId(ID);
ID.AddInteger(TargetFlags);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
ID.AddInteger(Offset);
ID.AddInteger(TargetFlags);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
- SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
- TargetFlags);
+ SDNode *N =
+ new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, TargetFlags);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
ID.AddPointer(MBB);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
return SDValue(N, 0);
}
+SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
+ SDNode *&N = MCSymbols[Sym];
+ if (N)
+ return SDValue(N, 0);
+ N = new (NodeAllocator) MCSymbolSDNode(Sym, VT);
+ InsertNode(N);
+ return SDValue(N, 0);
+}
+
SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
unsigned char TargetFlags) {
SDNode *&N =
// N2 to point at N1.
static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
std::swap(N1, N2);
- int NElts = M.size();
- for (int i = 0; i != NElts; ++i) {
- if (M[i] >= NElts)
- M[i] -= NElts;
- else if (M[i] >= 0)
- M[i] += NElts;
- }
+ ShuffleVectorSDNode::commuteMask(M);
}
SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
ID.AddInteger(MaskVec[i]);
void* IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0);
// Allocate the mask array for the node out of the BumpPtrAllocator, since
SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
MVT VT = SV.getSimpleValueType(0);
- unsigned NumElems = VT.getVectorNumElements();
- SmallVector<int, 8> MaskVec;
-
- for (unsigned i = 0; i != NumElems; ++i) {
- int Idx = SV.getMaskElt(i);
- if (Idx >= 0) {
- if (Idx < (int)NumElems)
- Idx += NumElems;
- else
- Idx -= NumElems;
- }
- MaskVec.push_back(Idx);
- }
+ SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
+ ShuffleVectorSDNode::commuteMask(MaskVec);
SDValue Op0 = SV.getOperand(0);
SDValue Op1 = SV.getOperand(1);
SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
void* IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0);
CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
ID.AddInteger(RegNo);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
ID.AddPointer(RegMask);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
ID.AddPointer(Label);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
ID.AddInteger(Offset);
ID.AddInteger(TargetFlags);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
ID.AddPointer(V);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
ID.AddPointer(MD);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
return SDValue(N, 0);
}
+SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
+ if (VT == V.getValueType())
+ return V;
+
+ return getNode(ISD::BITCAST, SDLoc(V), VT, V);
+}
+
/// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
unsigned SrcAS, unsigned DestAS) {
ID.AddInteger(DestAS);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
/// the target's desired shift amount type.
SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
EVT OpTy = Op.getValueType();
- EVT ShTy = TLI->getShiftAmountTy(LHSTy);
+ EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
if (OpTy == ShTy || OpTy.isVector()) return Op;
- ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
- return getNode(Opcode, SDLoc(Op), ShTy, Op);
+ return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
+}
+
+SDValue SelectionDAG::expandVAArg(SDNode *Node) {
+ SDLoc dl(Node);
+ const TargetLowering &TLI = getTargetLoweringInfo();
+ const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
+ EVT VT = Node->getValueType(0);
+ SDValue Tmp1 = Node->getOperand(0);
+ SDValue Tmp2 = Node->getOperand(1);
+ unsigned Align = Node->getConstantOperandVal(3);
+
+ SDValue VAListLoad =
+ getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1, Tmp2,
+ MachinePointerInfo(V), false, false, false, 0);
+ SDValue VAList = VAListLoad;
+
+ if (Align > TLI.getMinStackArgumentAlignment()) {
+ assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
+
+ VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
+ getConstant(Align - 1, dl, VAList.getValueType()));
+
+ VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList,
+ getConstant(-(int64_t)Align, dl, VAList.getValueType()));
+ }
+
+ // Increment the pointer, VAList, to the next vaarg
+ Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
+ getConstant(getDataLayout().getTypeAllocSize(
+ VT.getTypeForEVT(*getContext())),
+ dl, VAList.getValueType()));
+ // Store the incremented VAList to the legalized pointer
+ Tmp1 = getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2,
+ MachinePointerInfo(V), false, false, 0);
+ // Load the actual argument out of the pointer VAList
+ return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo(),
+ false, false, false, 0);
+}
+
+SDValue SelectionDAG::expandVACopy(SDNode *Node) {
+ SDLoc dl(Node);
+ const TargetLowering &TLI = getTargetLoweringInfo();
+ // This defaults to loading a pointer from the input and storing it to the
+ // output, returning the chain.
+ const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
+ const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
+ SDValue Tmp1 = getLoad(TLI.getPointerTy(getDataLayout()), dl,
+ Node->getOperand(0), Node->getOperand(2),
+ MachinePointerInfo(VS), false, false, false, 0);
+ return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
+ MachinePointerInfo(VD), false, false, 0);
}
/// CreateStackTemporary - Create a stack temporary, suitable for holding the
unsigned ByteSize = VT.getStoreSize();
Type *Ty = VT.getTypeForEVT(*getContext());
unsigned StackAlign =
- std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
+ std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
- return getFrameIndex(FrameIdx, TLI->getPointerTy());
+ return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
}
/// CreateStackTemporary - Create a stack temporary suitable for holding
/// either of the specified value types.
SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
- unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
- VT2.getStoreSizeInBits())/8;
+ unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
Type *Ty1 = VT1.getTypeForEVT(*getContext());
Type *Ty2 = VT2.getTypeForEVT(*getContext());
- const DataLayout *TD = TLI->getDataLayout();
- unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
- TD->getPrefTypeAlignment(Ty2));
+ const DataLayout &DL = getDataLayout();
+ unsigned Align =
+ std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
- return getFrameIndex(FrameIdx, TLI->getPointerTy());
+ return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
}
SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
switch (Cond) {
default: break;
case ISD::SETFALSE:
- case ISD::SETFALSE2: return getConstant(0, VT);
+ case ISD::SETFALSE2: return getConstant(0, dl, VT);
case ISD::SETTRUE:
case ISD::SETTRUE2: {
TargetLowering::BooleanContent Cnt =
TLI->getBooleanContents(N1->getValueType(0));
return getConstant(
- Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
+ Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, dl,
+ VT);
}
case ISD::SETOEQ:
break;
}
- if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
+ if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
const APInt &C2 = N2C->getAPIntValue();
- if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
+ if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
const APInt &C1 = N1C->getAPIntValue();
switch (Cond) {
default: llvm_unreachable("Unknown integer setcc!");
- case ISD::SETEQ: return getConstant(C1 == C2, VT);
- case ISD::SETNE: return getConstant(C1 != C2, VT);
- case ISD::SETULT: return getConstant(C1.ult(C2), VT);
- case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
- case ISD::SETULE: return getConstant(C1.ule(C2), VT);
- case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
- case ISD::SETLT: return getConstant(C1.slt(C2), VT);
- case ISD::SETGT: return getConstant(C1.sgt(C2), VT);
- case ISD::SETLE: return getConstant(C1.sle(C2), VT);
- case ISD::SETGE: return getConstant(C1.sge(C2), VT);
+ case ISD::SETEQ: return getConstant(C1 == C2, dl, VT);
+ case ISD::SETNE: return getConstant(C1 != C2, dl, VT);
+ case ISD::SETULT: return getConstant(C1.ult(C2), dl, VT);
+ case ISD::SETUGT: return getConstant(C1.ugt(C2), dl, VT);
+ case ISD::SETULE: return getConstant(C1.ule(C2), dl, VT);
+ case ISD::SETUGE: return getConstant(C1.uge(C2), dl, VT);
+ case ISD::SETLT: return getConstant(C1.slt(C2), dl, VT);
+ case ISD::SETGT: return getConstant(C1.sgt(C2), dl, VT);
+ case ISD::SETLE: return getConstant(C1.sle(C2), dl, VT);
+ case ISD::SETGE: return getConstant(C1.sge(C2), dl, VT);
}
}
}
- if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
- if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
+ if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1)) {
+ if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2)) {
APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
switch (Cond) {
default: break;
case ISD::SETEQ: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
// fall through
- case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
+ case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, dl, VT);
case ISD::SETNE: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
// fall through
case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
- R==APFloat::cmpLessThan, VT);
+ R==APFloat::cmpLessThan, dl, VT);
case ISD::SETLT: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
// fall through
- case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
+ case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, dl, VT);
case ISD::SETGT: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
// fall through
- case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
+ case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, dl, VT);
case ISD::SETLE: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
// fall through
case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
- R==APFloat::cmpEqual, VT);
+ R==APFloat::cmpEqual, dl, VT);
case ISD::SETGE: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
// fall through
case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
- R==APFloat::cmpEqual, VT);
- case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT);
- case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT);
+ R==APFloat::cmpEqual, dl, VT);
+ case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, dl, VT);
+ case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, dl, VT);
case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
- R==APFloat::cmpEqual, VT);
- case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
+ R==APFloat::cmpEqual, dl, VT);
+ case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, dl, VT);
case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
- R==APFloat::cmpLessThan, VT);
+ R==APFloat::cmpLessThan, dl, VT);
case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
- R==APFloat::cmpUnordered, VT);
- case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
- case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
+ R==APFloat::cmpUnordered, dl, VT);
+ case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, dl, VT);
+ case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, dl, VT);
}
} else {
// Ensure that the constant occurs on the RHS.
unsigned MemBits = VT.getScalarType().getSizeInBits();
KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
} else if (const MDNode *Ranges = LD->getRanges()) {
- computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
+ if (LD->getExtensionType() == ISD::NON_EXTLOAD)
+ computeKnownBitsFromRangeMetadata(*Ranges, KnownZero, KnownOne);
}
break;
}
// Output known-0 bits are known if clear or set in both the low clear bits
// common to both LHS & RHS. For example, 8+(X<<3) is known to have the
// low 3 bits clear.
+ // Output known-0 bits are also known if the top bits of each input are
+ // known to be clear. For example, if one input has the top 10 bits clear
+ // and the other has the top 8 bits clear, we know the top 7 bits of the
+ // output must be clear.
computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
- unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
+ unsigned KnownZeroHigh = KnownZero2.countLeadingOnes();
+ unsigned KnownZeroLow = KnownZero2.countTrailingOnes();
computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
- KnownZeroOut = std::min(KnownZeroOut,
+ KnownZeroHigh = std::min(KnownZeroHigh,
+ KnownZero2.countLeadingOnes());
+ KnownZeroLow = std::min(KnownZeroLow,
KnownZero2.countTrailingOnes());
if (Op.getOpcode() == ISD::ADD) {
- KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
+ KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroLow);
+ if (KnownZeroHigh > 1)
+ KnownZero |= APInt::getHighBitsSet(BitWidth, KnownZeroHigh - 1);
break;
}
// information if we know (at least) that the low two bits are clear. We
// then return to the caller that the low bit is unknown but that other bits
// are known zero.
- if (KnownZeroOut >= 2) // ADDE
- KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
+ if (KnownZeroLow >= 2) // ADDE
+ KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroLow);
break;
}
case ISD::SREM:
KnownOne = KnownOne.trunc(BitWidth);
break;
}
+ case ISD::SMIN:
+ case ISD::SMAX:
+ case ISD::UMIN:
+ case ISD::UMAX: {
+ APInt Op0Zero, Op0One;
+ APInt Op1Zero, Op1One;
+ computeKnownBits(Op.getOperand(0), Op0Zero, Op0One, Depth);
+ computeKnownBits(Op.getOperand(1), Op1Zero, Op1One, Depth);
+
+ KnownZero = Op0Zero & Op1Zero;
+ KnownOne = Op0One & Op1One;
+ break;
+ }
case ISD::FrameIndex:
case ISD::TargetFrameIndex:
if (unsigned Align = InferPtrAlignment(Op)) {
if (Tmp == 1) return 1; // Early out.
Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
return std::min(Tmp, Tmp2);
-
+ case ISD::SELECT_CC:
+ Tmp = ComputeNumSignBits(Op.getOperand(2), Depth+1);
+ if (Tmp == 1) return 1; // Early out.
+ Tmp2 = ComputeNumSignBits(Op.getOperand(3), Depth+1);
+ return std::min(Tmp, Tmp2);
+ case ISD::SMIN:
+ case ISD::SMAX:
+ case ISD::UMIN:
+ case ISD::UMAX:
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
+ if (Tmp == 1)
+ return 1; // Early out.
+ Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
+ return std::min(Tmp, Tmp2);
case ISD::SADDO:
case ISD::UADDO:
case ISD::SSUBO:
const int rIndex = Items - 1 -
cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
- // If the sign portion ends in our element the substraction gives correct
+ // If the sign portion ends in our element the subtraction gives correct
// result. Otherwise it gives either negative or > bitwidth result
return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
}
return false;
}
+bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
+ assert(A.getValueType() == B.getValueType() &&
+ "Values must have the same type");
+ APInt AZero, AOne;
+ APInt BZero, BOne;
+ computeKnownBits(A, AZero, AOne);
+ computeKnownBits(B, BZero, BOne);
+ return (AZero | BZero).isAllOnesValue();
+}
+
+static SDValue FoldCONCAT_VECTORS(SDLoc DL, EVT VT, ArrayRef<SDValue> Ops,
+ llvm::SelectionDAG &DAG) {
+ if (Ops.size() == 1)
+ return Ops[0];
+
+ // Concat of UNDEFs is UNDEF.
+ if (std::all_of(Ops.begin(), Ops.end(),
+ [](SDValue Op) { return Op.isUndef(); }))
+ return DAG.getUNDEF(VT);
+
+ // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified
+ // to one big BUILD_VECTOR.
+ // FIXME: Add support for UNDEF and SCALAR_TO_VECTOR as well.
+ if (!std::all_of(Ops.begin(), Ops.end(), [](SDValue Op) {
+ return Op.getOpcode() == ISD::BUILD_VECTOR;
+ }))
+ return SDValue();
+
+ EVT SVT = VT.getScalarType();
+ SmallVector<SDValue, 16> Elts;
+ for (SDValue Op : Ops)
+ Elts.append(Op->op_begin(), Op->op_end());
+
+ // BUILD_VECTOR requires all inputs to be of the same type, find the
+ // maximum type and extend them all.
+ for (SDValue Op : Elts)
+ SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
+
+ if (SVT.bitsGT(VT.getScalarType()))
+ for (SDValue &Op : Elts)
+ Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
+ ? DAG.getZExtOrTrunc(Op, DL, SVT)
+ : DAG.getSExtOrTrunc(Op, DL, SVT);
+
+ return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
+}
+
/// getNode - Gets or creates the specified node.
///
SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, getVTList(VT), None);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
// doesn't create new constants with different values. Nevertheless, the
// opaque flag is preserved during folding to prevent future folding with
// other constants.
- if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
const APInt &Val = C->getAPIntValue();
switch (Opcode) {
default: break;
case ISD::SIGN_EXTEND:
- return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
+ return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
C->isTargetOpcode(), C->isOpaque());
case ISD::ANY_EXTEND:
case ISD::ZERO_EXTEND:
case ISD::TRUNCATE:
- return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
+ return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
C->isTargetOpcode(), C->isOpaque());
case ISD::UINT_TO_FP:
case ISD::SINT_TO_FP: {
(void)apf.convertFromAPInt(Val,
Opcode==ISD::SINT_TO_FP,
APFloat::rmNearestTiesToEven);
- return getConstantFP(apf, VT);
+ return getConstantFP(apf, DL, VT);
}
case ISD::BITCAST:
if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
- return getConstantFP(APFloat(APFloat::IEEEhalf, Val), VT);
+ return getConstantFP(APFloat(APFloat::IEEEhalf, Val), DL, VT);
if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
- return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
- else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
- return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
+ return getConstantFP(APFloat(APFloat::IEEEsingle, Val), DL, VT);
+ if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
+ return getConstantFP(APFloat(APFloat::IEEEdouble, Val), DL, VT);
+ if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
+ return getConstantFP(APFloat(APFloat::IEEEquad, Val), DL, VT);
break;
case ISD::BSWAP:
- return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
+ return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
C->isOpaque());
case ISD::CTPOP:
- return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
+ return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
C->isOpaque());
case ISD::CTLZ:
case ISD::CTLZ_ZERO_UNDEF:
- return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
+ return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
C->isOpaque());
case ISD::CTTZ:
case ISD::CTTZ_ZERO_UNDEF:
- return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
+ return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
C->isOpaque());
}
}
// Constant fold unary operations with a floating point constant operand.
- if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
+ if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
APFloat V = C->getValueAPF(); // make copy
switch (Opcode) {
case ISD::FNEG:
V.changeSign();
- return getConstantFP(V, VT);
+ return getConstantFP(V, DL, VT);
case ISD::FABS:
V.clearSign();
- return getConstantFP(V, VT);
+ return getConstantFP(V, DL, VT);
case ISD::FCEIL: {
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
if (fs == APFloat::opOK || fs == APFloat::opInexact)
- return getConstantFP(V, VT);
+ return getConstantFP(V, DL, VT);
break;
}
case ISD::FTRUNC: {
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
if (fs == APFloat::opOK || fs == APFloat::opInexact)
- return getConstantFP(V, VT);
+ return getConstantFP(V, DL, VT);
break;
}
case ISD::FFLOOR: {
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
if (fs == APFloat::opOK || fs == APFloat::opInexact)
- return getConstantFP(V, VT);
+ return getConstantFP(V, DL, VT);
break;
}
case ISD::FP_EXTEND: {
// FIXME need to be more flexible about rounding mode.
(void)V.convert(EVTToAPFloatSemantics(VT),
APFloat::rmNearestTiesToEven, &ignored);
- return getConstantFP(V, VT);
+ return getConstantFP(V, DL, VT);
}
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT: {
if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual
break;
APInt api(VT.getSizeInBits(), x);
- return getConstant(api, VT);
+ return getConstant(api, DL, VT);
}
case ISD::BITCAST:
if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
- return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), VT);
+ return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
- return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
+ return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
- return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
+ return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
break;
}
}
- // Constant fold unary operations with a vector integer operand.
- if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
+ // Constant fold unary operations with a vector integer or float operand.
+ if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
if (BV->isConstant()) {
switch (Opcode) {
default:
// FIXME: Entirely reasonable to perform folding of other unary
// operations here as the need arises.
break;
+ case ISD::FNEG:
+ case ISD::FABS:
+ case ISD::FCEIL:
+ case ISD::FTRUNC:
+ case ISD::FFLOOR:
+ case ISD::FP_EXTEND:
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT:
+ case ISD::TRUNCATE:
case ISD::UINT_TO_FP:
- case ISD::SINT_TO_FP: {
- SmallVector<SDValue, 8> Ops;
- for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
- SDValue OpN = BV->getOperand(i);
- // Let the above scalar folding handle the conversion of each
- // element.
- OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
- OpN);
- Ops.push_back(OpN);
- }
- return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
+ case ISD::SINT_TO_FP:
+ case ISD::BSWAP:
+ case ISD::CTLZ:
+ case ISD::CTLZ_ZERO_UNDEF:
+ case ISD::CTTZ:
+ case ISD::CTTZ_ZERO_UNDEF:
+ case ISD::CTPOP: {
+ SDValue Ops = { Operand };
+ if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
+ return Fold;
}
}
}
VT.getVectorNumElements() ==
Operand.getValueType().getVectorNumElements()) &&
"Vector element count mismatch!");
+ assert(Operand.getValueType().bitsLT(VT) &&
+ "Invalid fpext node, dst < src!");
if (Operand.getOpcode() == ISD::UNDEF)
return getUNDEF(VT);
break;
assert(VT.isInteger() && Operand.getValueType().isInteger() &&
"Invalid SIGN_EXTEND!");
if (Operand.getValueType() == VT) return Operand; // noop extension
- assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
- "Invalid sext node, dst < src!");
assert((!VT.isVector() ||
VT.getVectorNumElements() ==
Operand.getValueType().getVectorNumElements()) &&
"Vector element count mismatch!");
+ assert(Operand.getValueType().bitsLT(VT) &&
+ "Invalid sext node, dst < src!");
if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
else if (OpOpcode == ISD::UNDEF)
// sext(undef) = 0, because the top bits will all be the same.
- return getConstant(0, VT);
+ return getConstant(0, DL, VT);
break;
case ISD::ZERO_EXTEND:
assert(VT.isInteger() && Operand.getValueType().isInteger() &&
"Invalid ZERO_EXTEND!");
if (Operand.getValueType() == VT) return Operand; // noop extension
- assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
- "Invalid zext node, dst < src!");
assert((!VT.isVector() ||
VT.getVectorNumElements() ==
Operand.getValueType().getVectorNumElements()) &&
"Vector element count mismatch!");
+ assert(Operand.getValueType().bitsLT(VT) &&
+ "Invalid zext node, dst < src!");
if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
return getNode(ISD::ZERO_EXTEND, DL, VT,
Operand.getNode()->getOperand(0));
else if (OpOpcode == ISD::UNDEF)
// zext(undef) = 0, because the top bits will be zero.
- return getConstant(0, VT);
+ return getConstant(0, DL, VT);
break;
case ISD::ANY_EXTEND:
assert(VT.isInteger() && Operand.getValueType().isInteger() &&
"Invalid ANY_EXTEND!");
if (Operand.getValueType() == VT) return Operand; // noop extension
- assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
- "Invalid anyext node, dst < src!");
assert((!VT.isVector() ||
VT.getVectorNumElements() ==
Operand.getValueType().getVectorNumElements()) &&
"Vector element count mismatch!");
+ assert(Operand.getValueType().bitsLT(VT) &&
+ "Invalid anyext node, dst < src!");
if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
OpOpcode == ISD::ANY_EXTEND)
assert(VT.isInteger() && Operand.getValueType().isInteger() &&
"Invalid TRUNCATE!");
if (Operand.getValueType() == VT) return Operand; // noop truncate
- assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
- "Invalid truncate node, src < dst!");
assert((!VT.isVector() ||
VT.getVectorNumElements() ==
Operand.getValueType().getVectorNumElements()) &&
"Vector element count mismatch!");
+ assert(Operand.getValueType().bitsGT(VT) &&
+ "Invalid truncate node, src < dst!");
if (OpOpcode == ISD::TRUNCATE)
return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
if (OpOpcode == ISD::UNDEF)
return getUNDEF(VT);
break;
+ case ISD::BSWAP:
+ assert(VT.isInteger() && VT == Operand.getValueType() &&
+ "Invalid BSWAP!");
+ assert((VT.getScalarSizeInBits() % 16 == 0) &&
+ "BSWAP types must be a multiple of 16 bits!");
+ if (OpOpcode == ISD::UNDEF)
+ return getUNDEF(VT);
+ break;
case ISD::BITCAST:
// Basic sanity checking.
assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
case ISD::FNEG:
// -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
+ // FIXME: FNEG has no fast-math-flags to propagate; use the FSUB's flags?
return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
- Operand.getNode()->getOperand(0));
+ Operand.getNode()->getOperand(0),
+ &cast<BinaryWithFlagsSDNode>(Operand.getNode())->Flags);
if (OpOpcode == ISD::FNEG) // --X -> X
return Operand.getNode()->getOperand(0);
break;
SDValue Ops[1] = { Operand };
AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
return SDValue(N, 0);
}
-SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
+static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
+ const APInt &C2) {
+ switch (Opcode) {
+ case ISD::ADD: return std::make_pair(C1 + C2, true);
+ case ISD::SUB: return std::make_pair(C1 - C2, true);
+ case ISD::MUL: return std::make_pair(C1 * C2, true);
+ case ISD::AND: return std::make_pair(C1 & C2, true);
+ case ISD::OR: return std::make_pair(C1 | C2, true);
+ case ISD::XOR: return std::make_pair(C1 ^ C2, true);
+ case ISD::SHL: return std::make_pair(C1 << C2, true);
+ case ISD::SRL: return std::make_pair(C1.lshr(C2), true);
+ case ISD::SRA: return std::make_pair(C1.ashr(C2), true);
+ case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
+ case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
+ case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true);
+ case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true);
+ case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true);
+ case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true);
+ case ISD::UDIV:
+ if (!C2.getBoolValue())
+ break;
+ return std::make_pair(C1.udiv(C2), true);
+ case ISD::UREM:
+ if (!C2.getBoolValue())
+ break;
+ return std::make_pair(C1.urem(C2), true);
+ case ISD::SDIV:
+ if (!C2.getBoolValue())
+ break;
+ return std::make_pair(C1.sdiv(C2), true);
+ case ISD::SREM:
+ if (!C2.getBoolValue())
+ break;
+ return std::make_pair(C1.srem(C2), true);
+ }
+ return std::make_pair(APInt(1, 0), false);
+}
+
+SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
+ const ConstantSDNode *Cst1,
+ const ConstantSDNode *Cst2) {
+ if (Cst1->isOpaque() || Cst2->isOpaque())
+ return SDValue();
+
+ std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
+ Cst2->getAPIntValue());
+ if (!Folded.second)
+ return SDValue();
+ return getConstant(Folded.first, DL, VT);
+}
+
+SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
SDNode *Cst1, SDNode *Cst2) {
// If the opcode is a target-specific ISD node, there's nothing we can
// do here and the operand rules may not line up with the below, so
if (Opcode >= ISD::BUILTIN_OP_END)
return SDValue();
- SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
- SmallVector<SDValue, 4> Outputs;
- EVT SVT = VT.getScalarType();
+ // Handle the case of two scalars.
+ if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) {
+ if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) {
+ if (SDValue Folded =
+ FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2)) {
+ if (!VT.isVector())
+ return Folded;
+ SmallVector<SDValue, 4> Outputs;
+ // We may have a vector type but a scalar result. Create a splat.
+ Outputs.resize(VT.getVectorNumElements(), Outputs.back());
+ // Build a big vector out of the scalar elements we generated.
+ return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
+ } else {
+ return SDValue();
+ }
+ }
+ }
- ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
- ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
- if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
+ // For vectors extract each constant element into Inputs so we can constant
+ // fold them individually.
+ BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
+ BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
+ if (!BV1 || !BV2)
return SDValue();
- if (Scalar1 && Scalar2)
- // Scalar instruction.
- Inputs.push_back(std::make_pair(Scalar1, Scalar2));
- else {
- // For vectors extract each constant element into Inputs so we can constant
- // fold them individually.
- BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
- BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
- if (!BV1 || !BV2)
+ assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
+
+ EVT SVT = VT.getScalarType();
+ SmallVector<SDValue, 4> Outputs;
+ for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
+ ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
+ ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
+ if (!V1 || !V2) // Not a constant, bail.
return SDValue();
- assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
+ if (V1->isOpaque() || V2->isOpaque())
+ return SDValue();
- for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
- ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
- ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
- if (!V1 || !V2) // Not a constant, bail.
- return SDValue();
+ // Avoid BUILD_VECTOR nodes that perform implicit truncation.
+ // FIXME: This is valid and could be handled by truncating the APInts.
+ if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
+ return SDValue();
- if (V1->isOpaque() || V2->isOpaque())
- return SDValue();
+ // Fold one vector element.
+ std::pair<APInt, bool> Folded = FoldValue(Opcode, V1->getAPIntValue(),
+ V2->getAPIntValue());
+ if (!Folded.second)
+ return SDValue();
+ Outputs.push_back(getConstant(Folded.first, DL, SVT));
+ }
- // Avoid BUILD_VECTOR nodes that perform implicit truncation.
- // FIXME: This is valid and could be handled by truncating the APInts.
- if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
- return SDValue();
+ assert(VT.getVectorNumElements() == Outputs.size() &&
+ "Vector size mismatch!");
- Inputs.push_back(std::make_pair(V1, V2));
- }
- }
+ // We may have a vector type but a scalar result. Create a splat.
+ Outputs.resize(VT.getVectorNumElements(), Outputs.back());
- // We have a number of constant values, constant fold them element by element.
- for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
- const APInt &C1 = Inputs[I].first->getAPIntValue();
- const APInt &C2 = Inputs[I].second->getAPIntValue();
+ // Build a big vector out of the scalar elements we generated.
+ return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
+}
- switch (Opcode) {
- case ISD::ADD:
- Outputs.push_back(getConstant(C1 + C2, SVT));
- break;
- case ISD::SUB:
- Outputs.push_back(getConstant(C1 - C2, SVT));
- break;
- case ISD::MUL:
- Outputs.push_back(getConstant(C1 * C2, SVT));
- break;
- case ISD::UDIV:
- if (!C2.getBoolValue())
- return SDValue();
- Outputs.push_back(getConstant(C1.udiv(C2), SVT));
- break;
- case ISD::UREM:
- if (!C2.getBoolValue())
- return SDValue();
- Outputs.push_back(getConstant(C1.urem(C2), SVT));
- break;
- case ISD::SDIV:
- if (!C2.getBoolValue())
- return SDValue();
- Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
- break;
- case ISD::SREM:
- if (!C2.getBoolValue())
- return SDValue();
- Outputs.push_back(getConstant(C1.srem(C2), SVT));
- break;
- case ISD::AND:
- Outputs.push_back(getConstant(C1 & C2, SVT));
- break;
- case ISD::OR:
- Outputs.push_back(getConstant(C1 | C2, SVT));
- break;
- case ISD::XOR:
- Outputs.push_back(getConstant(C1 ^ C2, SVT));
- break;
- case ISD::SHL:
- Outputs.push_back(getConstant(C1 << C2, SVT));
- break;
- case ISD::SRL:
- Outputs.push_back(getConstant(C1.lshr(C2), SVT));
- break;
- case ISD::SRA:
- Outputs.push_back(getConstant(C1.ashr(C2), SVT));
- break;
- case ISD::ROTL:
- Outputs.push_back(getConstant(C1.rotl(C2), SVT));
- break;
- case ISD::ROTR:
- Outputs.push_back(getConstant(C1.rotr(C2), SVT));
- break;
- default:
+SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode, SDLoc DL,
+ EVT VT,
+ ArrayRef<SDValue> Ops,
+ const SDNodeFlags *Flags) {
+ // If the opcode is a target-specific ISD node, there's nothing we can
+ // do here and the operand rules may not line up with the below, so
+ // bail early.
+ if (Opcode >= ISD::BUILTIN_OP_END)
+ return SDValue();
+
+ // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
+ if (!VT.isVector())
+ return SDValue();
+
+ unsigned NumElts = VT.getVectorNumElements();
+
+ auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
+ return !Op.getValueType().isVector() ||
+ Op.getValueType().getVectorNumElements() == NumElts;
+ };
+
+ auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
+ BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
+ return (Op.getOpcode() == ISD::UNDEF) ||
+ (Op.getOpcode() == ISD::CONDCODE) || (BV && BV->isConstant());
+ };
+
+ // All operands must be vector types with the same number of elements as
+ // the result type and must be either UNDEF or a build vector of constant
+ // or UNDEF scalars.
+ if (!std::all_of(Ops.begin(), Ops.end(), IsConstantBuildVectorOrUndef) ||
+ !std::all_of(Ops.begin(), Ops.end(), IsScalarOrSameVectorSize))
+ return SDValue();
+
+ // If we are comparing vectors, then the result needs to be a i1 boolean
+ // that is then sign-extended back to the legal result type.
+ EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
+
+ // Find legal integer scalar type for constant promotion and
+ // ensure that its scalar size is at least as large as source.
+ EVT LegalSVT = VT.getScalarType();
+ if (LegalSVT.isInteger()) {
+ LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
+ if (LegalSVT.bitsLT(SVT))
return SDValue();
- }
}
- assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
- "Expected a scalar or vector!"));
+ // Constant fold each scalar lane separately.
+ SmallVector<SDValue, 4> ScalarResults;
+ for (unsigned i = 0; i != NumElts; i++) {
+ SmallVector<SDValue, 4> ScalarOps;
+ for (SDValue Op : Ops) {
+ EVT InSVT = Op.getValueType().getScalarType();
+ BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
+ if (!InBV) {
+ // We've checked that this is UNDEF or a constant of some kind.
+ if (Op.isUndef())
+ ScalarOps.push_back(getUNDEF(InSVT));
+ else
+ ScalarOps.push_back(Op);
+ continue;
+ }
- // Handle the scalar case first.
- if (!VT.isVector())
- return Outputs.back();
+ SDValue ScalarOp = InBV->getOperand(i);
+ EVT ScalarVT = ScalarOp.getValueType();
- // We may have a vector type but a scalar result. Create a splat.
- Outputs.resize(VT.getVectorNumElements(), Outputs.back());
+ // Build vector (integer) scalar operands may need implicit
+ // truncation - do this before constant folding.
+ if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
+ ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
- // Build a big vector out of the scalar elements we generated.
- return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
+ ScalarOps.push_back(ScalarOp);
+ }
+
+ // Constant fold the scalar operands.
+ SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
+
+ // Legalize the (integer) scalar constant if necessary.
+ if (LegalSVT != SVT)
+ ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
+
+ // Scalar folding only succeeded if the result is a constant or UNDEF.
+ if (ScalarResult.getOpcode() != ISD::UNDEF &&
+ ScalarResult.getOpcode() != ISD::Constant &&
+ ScalarResult.getOpcode() != ISD::ConstantFP)
+ return SDValue();
+ ScalarResults.push_back(ScalarResult);
+ }
+
+ assert(ScalarResults.size() == NumElts &&
+ "Unexpected number of scalar results for BUILD_VECTOR");
+ return getNode(ISD::BUILD_VECTOR, DL, VT, ScalarResults);
}
SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
- SDValue N2, bool nuw, bool nsw, bool exact) {
- ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
- ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
+ SDValue N2, const SDNodeFlags *Flags) {
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
+ ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+ ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
+
+ // Canonicalize constant to RHS if commutative.
+ if (isCommutativeBinOp(Opcode)) {
+ if (N1C && !N2C) {
+ std::swap(N1C, N2C);
+ std::swap(N1, N2);
+ } else if (N1CFP && !N2CFP) {
+ std::swap(N1CFP, N2CFP);
+ std::swap(N1, N2);
+ }
+ }
+
switch (Opcode) {
default: break;
case ISD::TokenFactor:
if (N2.getOpcode() == ISD::EntryToken) return N1;
if (N1 == N2) return N1;
break;
- case ISD::CONCAT_VECTORS:
- // Concat of UNDEFs is UNDEF.
- if (N1.getOpcode() == ISD::UNDEF &&
- N2.getOpcode() == ISD::UNDEF)
- return getUNDEF(VT);
-
- // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
- // one big BUILD_VECTOR.
- if (N1.getOpcode() == ISD::BUILD_VECTOR &&
- N2.getOpcode() == ISD::BUILD_VECTOR) {
- SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
- N1.getNode()->op_end());
- Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
- return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
- }
+ case ISD::CONCAT_VECTORS: {
+ // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
+ SDValue Ops[] = {N1, N2};
+ if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
+ return V;
break;
+ }
case ISD::AND:
assert(VT.isInteger() && "This operator does not apply to FP types!");
assert(N1.getValueType() == N2.getValueType() &&
case ISD::MUL:
case ISD::SDIV:
case ISD::SREM:
+ case ISD::SMIN:
+ case ISD::SMAX:
+ case ISD::UMIN:
+ case ISD::UMAX:
assert(VT.isInteger() && "This operator does not apply to FP types!");
assert(N1.getValueType() == N2.getValueType() &&
N1.getValueType() == VT && "Binary operator types must match!");
case ISD::FREM:
if (getTarget().Options.UnsafeFPMath) {
if (Opcode == ISD::FADD) {
- // 0+x --> x
- if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
- if (CFP->getValueAPF().isZero())
- return N2;
// x+0 --> x
- if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
- if (CFP->getValueAPF().isZero())
- return N1;
+ if (N2CFP && N2CFP->getValueAPF().isZero())
+ return N1;
} else if (Opcode == ISD::FSUB) {
// x-0 --> x
- if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
- if (CFP->getValueAPF().isZero())
- return N1;
+ if (N2CFP && N2CFP->getValueAPF().isZero())
+ return N1;
} else if (Opcode == ISD::FMUL) {
- ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
- SDValue V = N2;
-
- // If the first operand isn't the constant, try the second
- if (!CFP) {
- CFP = dyn_cast<ConstantFPSDNode>(N2);
- V = N1;
- }
-
- if (CFP) {
- // 0*x --> 0
- if (CFP->isZero())
- return SDValue(CFP,0);
- // 1*x --> x
- if (CFP->isExactlyValue(1.0))
- return V;
- }
+ // x*0 --> 0
+ if (N2CFP && N2CFP->isZero())
+ return N2;
+ // x*1 --> x
+ if (N2CFP && N2CFP->isExactlyValue(1.0))
+ return N1;
}
}
assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
assert(VT.isFloatingPoint() &&
N1.getValueType().isFloatingPoint() &&
VT.bitsLE(N1.getValueType()) &&
- isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
+ N2C && "Invalid FP_ROUND!");
if (N1.getValueType() == VT) return N1; // noop conversion.
break;
case ISD::AssertSext:
assert(EVT.bitsLE(VT) && "Not extending!");
if (EVT == VT) return N1; // Not actually extending
+ auto SignExtendInReg = [&](APInt Val) {
+ unsigned FromBits = EVT.getScalarType().getSizeInBits();
+ Val <<= Val.getBitWidth() - FromBits;
+ Val = Val.ashr(Val.getBitWidth() - FromBits);
+ return getConstant(Val, DL, VT.getScalarType());
+ };
+
if (N1C) {
APInt Val = N1C->getAPIntValue();
- unsigned FromBits = EVT.getScalarType().getSizeInBits();
- Val <<= Val.getBitWidth()-FromBits;
- Val = Val.ashr(Val.getBitWidth()-FromBits);
- return getConstant(Val, VT);
+ return SignExtendInReg(Val);
+ }
+ if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
+ SmallVector<SDValue, 8> Ops;
+ for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
+ SDValue Op = N1.getOperand(i);
+ if (Op.getOpcode() == ISD::UNDEF) {
+ Ops.push_back(getUNDEF(VT.getScalarType()));
+ continue;
+ }
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
+ APInt Val = C->getAPIntValue();
+ Val = Val.zextOrTrunc(VT.getScalarSizeInBits());
+ Ops.push_back(SignExtendInReg(Val));
+ continue;
+ }
+ break;
+ }
+ if (Ops.size() == VT.getVectorNumElements())
+ return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
}
break;
}
if (N1.getOpcode() == ISD::UNDEF)
return getUNDEF(VT);
+ // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
+ if (N2C && N2C->getZExtValue() >= N1.getValueType().getVectorNumElements())
+ return getUNDEF(VT);
+
// EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
// expanding copies of large vectors from registers.
if (N2C &&
N1.getOperand(0).getValueType().getVectorNumElements();
return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
N1.getOperand(N2C->getZExtValue() / Factor),
- getConstant(N2C->getZExtValue() % Factor,
+ getConstant(N2C->getZExtValue() % Factor, DL,
N2.getValueType()));
}
// if the indices are known different, extract the element from
// the original vector.
SDValue N1Op2 = N1.getOperand(2);
- ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
+ ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
if (N1Op2C && N2C) {
if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
return N1.getOperand(N2C->getZExtValue());
// EXTRACT_ELEMENT of a constant int is also very common.
- if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
+ if (N1C) {
unsigned ElementSize = VT.getSizeInBits();
unsigned Shift = ElementSize * N2C->getZExtValue();
- APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
- return getConstant(ShiftedVal.trunc(ElementSize), VT);
+ APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
+ return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
}
break;
- case ISD::EXTRACT_SUBVECTOR: {
- SDValue Index = N2;
+ case ISD::EXTRACT_SUBVECTOR:
if (VT.isSimple() && N1.getValueType().isSimple()) {
assert(VT.isVector() && N1.getValueType().isVector() &&
"Extract subvector VTs must be a vectors!");
assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
"Extract subvector must be from larger vector to smaller vector!");
- if (isa<ConstantSDNode>(Index.getNode())) {
- assert((VT.getVectorNumElements() +
- cast<ConstantSDNode>(Index.getNode())->getZExtValue()
+ if (N2C) {
+ assert((VT.getVectorNumElements() + N2C->getZExtValue()
<= N1.getValueType().getVectorNumElements())
&& "Extract subvector overflow!");
}
}
break;
}
- }
// Perform trivial constant folding.
if (SDValue SV =
- FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()))
+ FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
return SV;
- // Canonicalize constant to RHS if commutative.
- if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
- std::swap(N1C, N2C);
- std::swap(N1, N2);
- }
-
// Constant fold FP operations.
bool HasFPExceptions = TLI->hasFloatingPointExceptions();
- ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
- ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
if (N1CFP) {
- if (!N2CFP && isCommutativeBinOp(Opcode)) {
- // Canonicalize constant to RHS if commutative.
- std::swap(N1CFP, N2CFP);
- std::swap(N1, N2);
- } else if (N2CFP) {
+ if (N2CFP) {
APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
APFloat::opStatus s;
switch (Opcode) {
case ISD::FADD:
s = V1.add(V2, APFloat::rmNearestTiesToEven);
if (!HasFPExceptions || s != APFloat::opInvalidOp)
- return getConstantFP(V1, VT);
+ return getConstantFP(V1, DL, VT);
break;
case ISD::FSUB:
s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
if (!HasFPExceptions || s!=APFloat::opInvalidOp)
- return getConstantFP(V1, VT);
+ return getConstantFP(V1, DL, VT);
break;
case ISD::FMUL:
s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
if (!HasFPExceptions || s!=APFloat::opInvalidOp)
- return getConstantFP(V1, VT);
+ return getConstantFP(V1, DL, VT);
break;
case ISD::FDIV:
s = V1.divide(V2, APFloat::rmNearestTiesToEven);
if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
s!=APFloat::opDivByZero)) {
- return getConstantFP(V1, VT);
+ return getConstantFP(V1, DL, VT);
}
break;
case ISD::FREM :
- s = V1.mod(V2, APFloat::rmNearestTiesToEven);
+ s = V1.mod(V2);
if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
s!=APFloat::opDivByZero)) {
- return getConstantFP(V1, VT);
+ return getConstantFP(V1, DL, VT);
}
break;
case ISD::FCOPYSIGN:
V1.copySign(V2);
- return getConstantFP(V1, VT);
+ return getConstantFP(V1, DL, VT);
default: break;
}
}
// FIXME need to be more flexible about rounding mode.
(void)V.convert(EVTToAPFloatSemantics(VT),
APFloat::rmNearestTiesToEven, &ignored);
- return getConstantFP(V, VT);
+ return getConstantFP(V, DL, VT);
}
}
case ISD::SRL:
case ISD::SHL:
if (!VT.isVector())
- return getConstant(0, VT); // fold op(undef, arg2) -> 0
+ return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
// For vectors, we can't easily build an all zero vector, just return
// the LHS.
return N2;
if (N1.getOpcode() == ISD::UNDEF)
// Handle undef ^ undef -> 0 special case. This is a common
// idiom (misuse).
- return getConstant(0, VT);
+ return getConstant(0, DL, VT);
// fallthrough
case ISD::ADD:
case ISD::ADDC:
case ISD::SRL:
case ISD::SHL:
if (!VT.isVector())
- return getConstant(0, VT); // fold op(arg1, undef) -> 0
+ return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
// For vectors, we can't easily build an all zero vector, just return
// the LHS.
return N1;
case ISD::OR:
if (!VT.isVector())
- return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
+ return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL, VT);
// For vectors, we can't easily build an all one vector, just return
// the LHS.
return N1;
// Memoize this node if possible.
BinarySDNode *N;
SDVTList VTs = getVTList(VT);
- const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
if (VT != MVT::Glue) {
SDValue Ops[] = {N1, N2};
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTs, Ops);
- if (BinOpHasFlags)
- AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
+ if (Flags)
+ E->intersectFlagsWith(Flags);
return SDValue(E, 0);
+ }
- N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
+ N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
CSEMap.InsertNode(N, IP);
} else {
-
- N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
+ N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
}
InsertNode(N);
SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
SDValue N1, SDValue N2, SDValue N3) {
// Perform various simplifications.
- ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
switch (Opcode) {
case ISD::FMA: {
ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
APFloat::opStatus s =
V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
- return getConstantFP(V1, VT);
+ return getConstantFP(V1, DL, VT);
}
break;
}
- case ISD::CONCAT_VECTORS:
- // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
- // one big BUILD_VECTOR.
- if (N1.getOpcode() == ISD::BUILD_VECTOR &&
- N2.getOpcode() == ISD::BUILD_VECTOR &&
- N3.getOpcode() == ISD::BUILD_VECTOR) {
- SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
- N1.getNode()->op_end());
- Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
- Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
- return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
- }
+ case ISD::CONCAT_VECTORS: {
+ // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
+ SDValue Ops[] = {N1, N2, N3};
+ if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
+ return V;
break;
+ }
case ISD::SETCC: {
// Use FoldSetCC to simplify SETCC's.
- SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
- if (Simp.getNode()) return Simp;
+ if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
+ return V;
+ // Vector constant folding.
+ SDValue Ops[] = {N1, N2, N3};
+ if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
+ return V;
break;
}
case ISD::SELECT:
- if (N1C) {
+ if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
if (N1C->getZExtValue())
return N2; // select true, X, Y -> X
return N3; // select false, X, Y -> Y
"Dest and insert subvector source types must match!");
assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
"Insert subvector must be from smaller vector to larger vector!");
- if (isa<ConstantSDNode>(Index.getNode())) {
+ if (isa<ConstantSDNode>(Index)) {
assert((N2.getValueType().getVectorNumElements() +
- cast<ConstantSDNode>(Index.getNode())->getZExtValue()
+ cast<ConstantSDNode>(Index)->getZExtValue()
<= VT.getVectorNumElements())
&& "Insert subvector overflow!");
}
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
assert(C->getAPIntValue().getBitWidth() == 8);
APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
if (VT.isInteger())
- return DAG.getConstant(Val, VT);
- return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
+ return DAG.getConstant(Val, dl, VT);
+ return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
+ VT);
}
- Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
+ assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
+ EVT IntVT = VT.getScalarType();
+ if (!IntVT.isInteger())
+ IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
+
+ Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
if (NumBits > 8) {
// Use a multiplication with 0x010101... to extend the input to the
// required length.
APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
- Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
+ Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
+ DAG.getConstant(Magic, dl, IntVT));
+ }
+
+ if (VT != Value.getValueType() && !VT.isInteger())
+ Value = DAG.getNode(ISD::BITCAST, dl, VT.getScalarType(), Value);
+ if (VT != Value.getValueType()) {
+ assert(VT.getVectorElementType() == Value.getValueType() &&
+ "value type should be one vector element here");
+ SmallVector<SDValue, 8> BVOps(VT.getVectorNumElements(), Value);
+ Value = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, BVOps);
}
return Value;
// Handle vector with all elements zero.
if (Str.empty()) {
if (VT.isInteger())
- return DAG.getConstant(0, VT);
+ return DAG.getConstant(0, dl, VT);
else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
- return DAG.getConstantFP(0.0, VT);
+ return DAG.getConstantFP(0.0, dl, VT);
else if (VT.isVector()) {
unsigned NumElts = VT.getVectorNumElements();
MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
return DAG.getNode(ISD::BITCAST, dl, VT,
- DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
- EltVT, NumElts)));
+ DAG.getConstant(0, dl,
+ EVT::getVectorVT(*DAG.getContext(),
+ EltVT, NumElts)));
} else
llvm_unreachable("Expected type!");
}
unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
APInt Val(NumVTBits, 0);
- if (TLI.isLittleEndian()) {
+ if (DAG.getDataLayout().isLittleEndian()) {
for (unsigned i = 0; i != NumBytes; ++i)
Val |= (uint64_t)(unsigned char)Str[i] << i*8;
} else {
// of a load, then it is cost effective to turn the load into the immediate.
Type *Ty = VT.getTypeForEVT(*DAG.getContext());
if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
- return DAG.getConstant(Val, VT);
+ return DAG.getConstant(Val, dl, VT);
return SDValue(nullptr, 0);
}
SelectionDAG &DAG) {
EVT VT = Base.getValueType();
return DAG.getNode(ISD::ADD, dl,
- VT, Base, DAG.getConstant(Offset, VT));
+ VT, Base, DAG.getConstant(Offset, dl, VT));
}
/// isMemSrcFromString - Returns true if memcpy source is a string constant.
return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
}
-/// FindOptimalMemOpLowering - Determines the optimial series memory ops
-/// to replace the memset / memcpy. Return true if the number of memory ops
-/// is below the threshold. It returns the types of the sequence of
-/// memory ops to perform memset / memcpy by reference.
+/// Determines the optimal series of memory ops to replace the memset / memcpy.
+/// Return true if the number of memory ops is below the threshold (Limit).
+/// It returns the types of the sequence of memory ops to perform
+/// memset / memcpy by reference.
static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
unsigned Limit, uint64_t Size,
unsigned DstAlign, unsigned SrcAlign,
if (VT == MVT::Other) {
unsigned AS = 0;
- if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
+ if (DstAlign >= DAG.getDataLayout().getPointerPrefAlignment(AS) ||
TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
- VT = TLI.getPointerTy();
+ VT = TLI.getPointerTy(DAG.getDataLayout());
} else {
switch (DstAlign & 7) {
case 0: VT = MVT::i64; break;
return true;
}
+static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
+ // On Darwin, -Os means optimize for size without hurting performance, so
+ // only really optimize for size when -Oz (MinSize) is used.
+ if (MF.getTarget().getTargetTriple().isOSDarwin())
+ return MF.getFunction()->optForMinSize();
+ return MF.getFunction()->optForSize();
+}
+
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
SDValue Chain, SDValue Dst,
SDValue Src, uint64_t Size,
bool DstAlignCanChange = false;
MachineFunction &MF = DAG.getMachineFunction();
MachineFrameInfo *MFI = MF.getFrameInfo();
- bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
+ bool OptSize = shouldLowerMemFuncForSize(MF);
FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
DstAlignCanChange = true;
if (DstAlignCanChange) {
Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
- unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
+ unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
// Don't promote to an alignment that would require dynamic stack
// realignment.
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
if (!TRI->needsStackRealignment(MF))
- while (NewAlign > Align &&
- TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
+ while (NewAlign > Align &&
+ DAG.getDataLayout().exceedsNaturalStackAlignment(NewAlign))
NewAlign /= 2;
if (NewAlign > Align) {
bool DstAlignCanChange = false;
MachineFunction &MF = DAG.getMachineFunction();
MachineFrameInfo *MFI = MF.getFrameInfo();
- bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
+ bool OptSize = shouldLowerMemFuncForSize(MF);
FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
DstAlignCanChange = true;
if (DstAlignCanChange) {
Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
- unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
+ unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
if (NewAlign > Align) {
// Give the stack frame object a larger alignment if needed.
if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
bool DstAlignCanChange = false;
MachineFunction &MF = DAG.getMachineFunction();
MachineFrameInfo *MFI = MF.getFrameInfo();
- bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
+ bool OptSize = shouldLowerMemFuncForSize(MF);
FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
DstAlignCanChange = true;
if (DstAlignCanChange) {
Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
- unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
+ unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
if (NewAlign > Align) {
// Give the stack frame object a larger alignment if needed.
if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
}
+static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
+ unsigned AS) {
+ // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
+ // pointer operands can be losslessly bitcasted to pointers of address space 0
+ if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) {
+ report_fatal_error("cannot lower memory intrinsic in address space " +
+ Twine(AS));
+ }
+}
+
SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
SDValue Src, SDValue Size,
unsigned Align, bool isVol, bool AlwaysInline,
- MachinePointerInfo DstPtrInfo,
+ bool isTailCall, MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo) {
assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
true, DstPtrInfo, SrcPtrInfo);
}
+ checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
+ checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
+
// FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
// memcpy is not guaranteed to be safe. libc memcpys aren't required to
// respect volatile, so they may do things like read or write memory
// Emit a library call.
TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
- Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
+ Entry.Ty = getDataLayout().getIntPtrType(*getContext());
Entry.Node = Dst; Args.push_back(Entry);
Entry.Node = Src; Args.push_back(Entry);
Entry.Node = Size; Args.push_back(Entry);
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
- CLI.setDebugLoc(dl).setChain(Chain)
- .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
- Type::getVoidTy(*getContext()),
- getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
- TLI->getPointerTy()), std::move(Args), 0)
- .setDiscardResult();
- std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
+ CLI.setDebugLoc(dl)
+ .setChain(Chain)
+ .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
+ Type::getVoidTy(*getContext()),
+ getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
+ TLI->getPointerTy(getDataLayout())),
+ std::move(Args), 0)
+ .setDiscardResult()
+ .setTailCall(isTailCall);
+ std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
}
SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
SDValue Src, SDValue Size,
- unsigned Align, bool isVol,
+ unsigned Align, bool isVol, bool isTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo) {
assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
return Result;
}
+ checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
+ checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
+
// FIXME: If the memmove is volatile, lowering it to plain libc memmove may
// not be safe. See memcpy above for more details.
// Emit a library call.
TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
- Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
+ Entry.Ty = getDataLayout().getIntPtrType(*getContext());
Entry.Node = Dst; Args.push_back(Entry);
Entry.Node = Src; Args.push_back(Entry);
Entry.Node = Size; Args.push_back(Entry);
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
- CLI.setDebugLoc(dl).setChain(Chain)
- .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
- Type::getVoidTy(*getContext()),
- getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
- TLI->getPointerTy()), std::move(Args), 0)
- .setDiscardResult();
- std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
+ CLI.setDebugLoc(dl)
+ .setChain(Chain)
+ .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
+ Type::getVoidTy(*getContext()),
+ getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
+ TLI->getPointerTy(getDataLayout())),
+ std::move(Args), 0)
+ .setDiscardResult()
+ .setTailCall(isTailCall);
+ std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
}
SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
SDValue Src, SDValue Size,
- unsigned Align, bool isVol,
+ unsigned Align, bool isVol, bool isTailCall,
MachinePointerInfo DstPtrInfo) {
assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
return Result;
}
+ checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
+
// Emit a library call.
- Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
+ Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
Entry.Node = Dst; Entry.Ty = IntPtrTy;
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
- CLI.setDebugLoc(dl).setChain(Chain)
- .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
- Type::getVoidTy(*getContext()),
- getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
- TLI->getPointerTy()), std::move(Args), 0)
- .setDiscardResult();
+ CLI.setDebugLoc(dl)
+ .setChain(Chain)
+ .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
+ Type::getVoidTy(*getContext()),
+ getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
+ TLI->getPointerTy(getDataLayout())),
+ std::move(Args), 0)
+ .setDiscardResult()
+ .setTailCall(isTailCall);
std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
AddNodeIDNode(ID, Opcode, VTList, Ops);
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void* IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
cast<AtomicSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
AddNodeIDNode(ID, Opcode, VTList, Ops);
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
/// MachinePointerInfo record from it. This is particularly useful because the
/// code generator has many cases where it doesn't bother passing in a
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
-static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
+static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr,
+ int64_t Offset = 0) {
// If this is FI+Offset, we can model it.
if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
- return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
+ return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
+ FI->getIndex(), Offset);
// If this is (FI+Offset1)+Offset2, we can model it.
if (Ptr.getOpcode() != ISD::ADD ||
return MachinePointerInfo();
int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
- return MachinePointerInfo::getFixedStack(FI, Offset+
- cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
+ return MachinePointerInfo::getFixedStack(
+ DAG.getMachineFunction(), FI,
+ Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
}
/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
/// MachinePointerInfo record from it. This is particularly useful because the
/// code generator has many cases where it doesn't bother passing in a
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
-static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
+static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr,
+ SDValue OffsetOp) {
// If the 'Offset' value isn't a constant, we can't handle this.
if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
- return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
+ return InferPointerInfo(DAG, Ptr, OffsetNode->getSExtValue());
if (OffsetOp.getOpcode() == ISD::UNDEF)
- return InferPointerInfo(Ptr);
+ return InferPointerInfo(DAG, Ptr);
return MachinePointerInfo();
}
// If we don't have a PtrInfo, infer the trivial frame index case to simplify
// clients.
if (PtrInfo.V.isNull())
- PtrInfo = InferPointerInfo(Ptr, Offset);
+ PtrInfo = InferPointerInfo(*this, Ptr, Offset);
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO =
MMO->isInvariant()));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
cast<LoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
Flags |= MachineMemOperand::MONonTemporal;
if (PtrInfo.V.isNull())
- PtrInfo = InferPointerInfo(Ptr);
+ PtrInfo = InferPointerInfo(*this, Ptr);
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO =
MMO->isNonTemporal(), MMO->isInvariant()));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
Flags |= MachineMemOperand::MONonTemporal;
if (PtrInfo.V.isNull())
- PtrInfo = InferPointerInfo(Ptr);
+ PtrInfo = InferPointerInfo(*this, Ptr);
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO =
MMO->isNonTemporal(), MMO->isInvariant()));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
ID.AddInteger(ST->getRawSubclassData());
ID.AddInteger(ST->getPointerInfo().getAddrSpace());
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0);
SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
MMO->isInvariant()));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
MMO->isNonTemporal(), MMO->isInvariant()));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
return SDValue(N, 0);
}
+SDValue
+SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
+ ArrayRef<SDValue> Ops,
+ MachineMemOperand *MMO) {
+
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
+ ID.AddInteger(VT.getRawBits());
+ ID.AddInteger(encodeMemSDNodeFlags(ISD::NON_EXTLOAD, ISD::UNINDEXED,
+ MMO->isVolatile(),
+ MMO->isNonTemporal(),
+ MMO->isInvariant()));
+ ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
+ void *IP = nullptr;
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
+ cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
+ return SDValue(E, 0);
+ }
+ MaskedGatherSDNode *N =
+ new (NodeAllocator) MaskedGatherSDNode(dl.getIROrder(), dl.getDebugLoc(),
+ Ops, VTs, VT, MMO);
+ CSEMap.InsertNode(N, IP);
+ InsertNode(N);
+ return SDValue(N, 0);
+}
+
+SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
+ ArrayRef<SDValue> Ops,
+ MachineMemOperand *MMO) {
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
+ ID.AddInteger(VT.getRawBits());
+ ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
+ MMO->isNonTemporal(),
+ MMO->isInvariant()));
+ ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
+ void *IP = nullptr;
+ if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
+ cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
+ return SDValue(E, 0);
+ }
+ SDNode *N =
+ new (NodeAllocator) MaskedScatterSDNode(dl.getIROrder(), dl.getDebugLoc(),
+ Ops, VTs, VT, MMO);
+ CSEMap.InsertNode(N, IP);
+ InsertNode(N);
+ return SDValue(N, 0);
+}
+
SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
SDValue Chain, SDValue Ptr,
SDValue SV,
unsigned Align) {
- SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
+ SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
}
}
SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
- ArrayRef<SDValue> Ops) {
+ ArrayRef<SDValue> Ops, const SDNodeFlags *Flags) {
unsigned NumOps = Ops.size();
switch (NumOps) {
case 0: return getNode(Opcode, DL, VT);
case 1: return getNode(Opcode, DL, VT, Ops[0]);
- case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
+ case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
default: break;
}
switch (Opcode) {
default: break;
+ case ISD::CONCAT_VECTORS: {
+ // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
+ if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
+ return V;
+ break;
+ }
case ISD::SELECT_CC: {
assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
assert(Ops[0].getValueType() == Ops[1].getValueType() &&
AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
if (NumOps == 1) {
"Update with wrong number of operands");
// If no operands changed just return the input node.
- if (!Ops.empty() && std::equal(Ops.begin(), Ops.end(), N->op_begin()))
+ if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
return N;
// See if the modified node already exists.
/// For IROrder, we keep the smaller of the two
SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
DebugLoc NLoc = N->getDebugLoc();
- if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
- (OLoc.getDebugLoc() != NLoc)) {
+ if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
N->setDebugLoc(DebugLoc());
}
unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opc, VTs, Ops);
- if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *ON = FindNodeOrInsertPos(ID, N->getDebugLoc(), IP))
return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
}
FoldingSetNodeID ID;
AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
}
}
SDValue
SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
SDValue Operand) {
- SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
+ SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
VT, Operand, SRIdxVal);
return SDValue(Subreg, 0);
SDValue
SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
SDValue Operand, SDValue Subreg) {
- SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
+ SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
VT, Operand, Subreg, SRIdxVal);
return SDValue(Result, 0);
/// getNodeIfExists - Get the specified node if it's already available, or
/// else return NULL.
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
- ArrayRef<SDValue> Ops, bool nuw, bool nsw,
- bool exact) {
+ ArrayRef<SDValue> Ops,
+ const SDNodeFlags *Flags) {
if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops);
- if (isBinOpWithFlags(Opcode))
- AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
void *IP = nullptr;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = FindNodeOrInsertPos(ID, DebugLoc(), IP)) {
+ if (Flags)
+ E->intersectFlagsWith(Flags);
return E;
+ }
}
return nullptr;
}
SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
unsigned R, bool IsIndirect, uint64_t Off,
DebugLoc DL, unsigned O) {
- return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
+ assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+ "Expected inlined-at fields to agree");
+ return new (DbgInfo->getAlloc())
+ SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
}
/// Constant
SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
const Value *C, uint64_t Off,
DebugLoc DL, unsigned O) {
- return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
+ assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+ "Expected inlined-at fields to agree");
+ return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, Off, DL, O);
}
/// FrameIndex
SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
unsigned FI, uint64_t Off,
DebugLoc DL, unsigned O) {
- return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
+ assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+ "Expected inlined-at fields to agree");
+ return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, FI, Off, DL, O);
}
namespace {
// Node Id fields for nodes At SortedPos and after will contain the
// count of outstanding operands.
for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
- SDNode *N = I++;
+ SDNode *N = &*I++;
checkForCycles(N, this);
unsigned Degree = N->getNumOperands();
if (Degree == 0) {
// A node with no uses, add it to the result array immediately.
N->setNodeId(DAGSize++);
- allnodes_iterator Q = N;
+ allnodes_iterator Q(N);
if (Q != SortedPos)
SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
assert(SortedPos != AllNodes.end() && "Overran node list");
// Visit all the nodes. As we iterate, move nodes into sorted order,
// such that by the time the end is reached all nodes will be sorted.
- for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
- SDNode *N = I;
+ for (SDNode &Node : allnodes()) {
+ SDNode *N = &Node;
checkForCycles(N, this);
// N is in sorted position, so all its uses have one less operand
// that needs to be sorted.
P->setNodeId(Degree);
}
}
- if (I == SortedPos) {
+ if (&Node == SortedPos) {
#ifndef NDEBUG
- SDNode *S = ++I;
+ allnodes_iterator I(N);
+ SDNode *S = &*++I;
dbgs() << "Overran sorted position:\n";
S->dumprFull(this); dbgs() << "\n";
dbgs() << "Checking if this is due to cycles\n";
// SDNode Class
//===----------------------------------------------------------------------===//
+bool llvm::isNullConstant(SDValue V) {
+ ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
+ return Const != nullptr && Const->isNullValue();
+}
+
+bool llvm::isNullFPConstant(SDValue V) {
+ ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
+ return Const != nullptr && Const->isZero() && !Const->isNegative();
+}
+
+bool llvm::isAllOnesConstant(SDValue V) {
+ ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
+ return Const != nullptr && Const->isAllOnesValue();
+}
+
+bool llvm::isOneConstant(SDValue V) {
+ ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
+ return Const != nullptr && Const->isOne();
+}
+
HandleSDNode::~HandleSDNode() {
DropOperands();
}
/// isOnlyUserOf - Return true if this node is the only use of N.
///
-bool SDNode::isOnlyUserOf(SDNode *N) const {
+bool SDNode::isOnlyUserOf(const SDNode *N) const {
bool Seen = false;
for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
SDNode *User = *I;
/// isOperand - Return true if this node is an operand of N.
///
-bool SDValue::isOperandOf(SDNode *N) const {
- for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
- if (*this == N->getOperand(i))
+bool SDValue::isOperandOf(const SDNode *N) const {
+ for (const SDValue &Op : N->op_values())
+ if (*this == Op)
return true;
return false;
}
-bool SDNode::isOperandOf(SDNode *N) const {
- for (unsigned i = 0, e = N->NumOperands; i != e; ++i)
- if (this == N->OperandList[i].getNode())
+bool SDNode::isOperandOf(const SDNode *N) const {
+ for (const SDValue &Op : N->op_values())
+ if (this == Op.getNode())
return true;
return false;
}
// Haven't visited N yet. Continue the search.
while (!Worklist.empty()) {
const SDNode *M = Worklist.pop_back_val();
- for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
- SDNode *Op = M->getOperand(i).getNode();
+ for (const SDValue &OpV : M->op_values()) {
+ SDNode *Op = OpV.getNode();
if (Visited.insert(Op).second)
Worklist.push_back(Op);
if (Op == N)
return cast<ConstantSDNode>(OperandList[Num])->getZExtValue();
}
+const SDNodeFlags *SDNode::getFlags() const {
+ if (auto *FlagsNode = dyn_cast<BinaryWithFlagsSDNode>(this))
+ return &FlagsNode->Flags;
+ return nullptr;
+}
+
+void SDNode::intersectFlagsWith(const SDNodeFlags *Flags) {
+ if (auto *FlagsNode = dyn_cast<BinaryWithFlagsSDNode>(this))
+ FlagsNode->Flags.intersectWith(Flags);
+}
+
SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
assert(N->getNumValues() == 1 &&
"Can't unroll a vector with multiple results!");
if (OperandVT.isVector()) {
// A vector operand; extract a single element.
EVT OperandEltVT = OperandVT.getVectorElementType();
- Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl,
- OperandEltVT,
- Operand,
- getConstant(i, TLI->getVectorIdxTy()));
+ Operands[j] =
+ getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
+ getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
} else {
// A scalar operand; just use it as is.
Operands[j] = Operand;
}
switch (N->getOpcode()) {
- default:
- Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands));
+ default: {
+ Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
+ N->getFlags()));
break;
+ }
case ISD::VSELECT:
Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
break;
const GlobalValue *GV;
int64_t GVOffset = 0;
if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
- unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType());
+ unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0);
- llvm::computeKnownBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne,
- TLI->getDataLayout());
+ llvm::computeKnownBits(const_cast<GlobalValue *>(GV), KnownZero, KnownOne,
+ getDataLayout());
unsigned AlignBits = KnownZero.countTrailingOnes();
unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
if (Align)
"More vector elements requested than available!");
SDValue Lo, Hi;
Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
- getConstant(0, TLI->getVectorIdxTy()));
+ getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
- getConstant(LoVT.getVectorNumElements(), TLI->getVectorIdxTy()));
+ getConstant(LoVT.getVectorNumElements(), DL,
+ TLI->getVectorIdxTy(getDataLayout())));
return std::make_pair(Lo, Hi);
}
Count = VT.getVectorNumElements();
EVT EltVT = VT.getVectorElementType();
- EVT IdxTy = TLI->getVectorIdxTy();
+ EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
SDLoc SL(Op);
for (unsigned i = Start, e = Start + Count; i != e; ++i) {
Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
- Op, getConstant(i, IdxTy)));
+ Op, getConstant(i, SL, IdxTy)));
}
}
ConstantSDNode *
BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
- return dyn_cast_or_null<ConstantSDNode>(
- getSplatValue(UndefElements).getNode());
+ return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
}
ConstantFPSDNode *
BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
- return dyn_cast_or_null<ConstantFPSDNode>(
- getSplatValue(UndefElements).getNode());
+ return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
+}
+
+int32_t
+BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
+ uint32_t BitWidth) const {
+ if (ConstantFPSDNode *CN =
+ dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
+ bool IsExact;
+ APSInt IntVal(BitWidth);
+ APFloat APF = CN->getValueAPF();
+ if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
+ APFloat::opOK ||
+ !IsExact)
+ return -1;
+
+ return IntVal.exactLogBase2();
+ }
+ return -1;
}
bool BuildVectorSDNode::isConstant() const {
- for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
- unsigned Opc = getOperand(i).getOpcode();
+ for (const SDValue &Op : op_values()) {
+ unsigned Opc = Op.getOpcode();
if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
return false;
}
abort();
}
- for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
- checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked, DAG);
+ for (const SDValue &Op : N->op_values())
+ checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
Checked.insert(N);
Visited.erase(N);
projects / oota-llvm.git / blobdiff