This is an updated version of the patch that was checked in at:
http://reviews.llvm.org/rL237046
but subsequently reverted because it exposed a bug in the DAG Combiner:
http://reviews.llvm.org/D9893
This time, there's an enablement flag ("EnableFMFInDAG") around the code in
SelectionDAGBuilder where we copy the set of FP optimization flags from IR
instructions to DAG nodes. So, in theory, there should be no functional change
from this patch as-is, but it will allow testing with the added functionality
to proceed via "-enable-fmf-dag" passed to llc.
This patch adds the minimum plumbing necessary to use IR-level
fast-math-flags (FMF) in the backend without actually using
them for anything yet. This is a follow-on to:
http://reviews.llvm.org/rL235997
Differential Revision: http://reviews.llvm.org/D10403
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@239828
91177308-0d34-0410-b5e6-
96231b3b80d8
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT);
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N);
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT);
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N);
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
- bool nuw = false, bool nsw = false, bool exact = false);
+ const SDNodeFlags *Flags = nullptr);
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
SDValue N3);
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
SDValue N3);
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
/// Get the specified node if it's already available, or else return NULL.
SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef<SDValue> Ops,
/// Get the specified node if it's already available, or else return NULL.
SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef<SDValue> Ops,
- bool nuw = false, bool nsw = false,
- bool exact = false);
+ const SDNodeFlags *Flags = nullptr);
/// Creates a SDDbgValue node.
SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R,
/// Creates a SDDbgValue node.
SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R,
void allnodes_clear();
BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
void allnodes_clear();
BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
- SDValue N1, SDValue N2, bool nuw, bool nsw,
- bool exact);
+ SDValue N1, SDValue N2,
+ const SDNodeFlags *Flags = nullptr);
/// Look up the node specified by ID in CSEMap. If it exists, return it. If
/// not, return the insertion token that will make insertion faster. This
/// Look up the node specified by ID in CSEMap. If it exists, return it. If
/// not, return the insertion token that will make insertion faster. This
case ISD::ADD:
case ISD::SUB:
case ISD::SHL:
case ISD::ADD:
case ISD::SUB:
case ISD::SHL:
+ case ISD::FADD:
+ case ISD::FDIV:
+ case ISD::FMUL:
+ case ISD::FREM:
+ case ISD::FSUB:
return true;
default:
return false;
return true;
default:
return false;
public:
SDNodeFlags Flags;
BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
public:
SDNodeFlags Flags;
BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
- SDValue X, SDValue Y)
- : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags() {}
+ SDValue X, SDValue Y, const SDNodeFlags &NodeFlags)
+ : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {}
static bool classof(const SDNode *N) {
return isBinOpWithFlags(N->getOpcode());
}
static bool classof(const SDNode *N) {
return isBinOpWithFlags(N->getOpcode());
}
if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
SDValue Ops[] = {N1, N0};
SDNode *CSENode;
if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
SDValue Ops[] = {N1, N0};
SDNode *CSENode;
- if (const BinaryWithFlagsSDNode *BinNode =
- dyn_cast<BinaryWithFlagsSDNode>(N)) {
+ if (const auto *BinNode = dyn_cast<BinaryWithFlagsSDNode>(N)) {
CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
- BinNode->Flags.hasNoUnsignedWrap(),
- BinNode->Flags.hasNoSignedWrap(),
- BinNode->Flags.hasExact());
} else {
CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops);
}
} else {
CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops);
}
ID.AddInteger(Op.getResNo());
}
}
ID.AddInteger(Op.getResNo());
}
}
+/// Add logical or fast math flag values to FoldingSetNodeID value.
+static void AddNodeIDFlags(FoldingSetNodeID &ID, unsigned Opcode,
+ const SDNodeFlags *Flags) {
+ if (!Flags || !isBinOpWithFlags(Opcode))
+ return;
-static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
- bool exact) {
- ID.AddBoolean(nuw);
- ID.AddBoolean(nsw);
- ID.AddBoolean(exact);
+ unsigned RawFlags = Flags->getRawFlags();
+ // If no flags are set, do not alter the ID. We must match the ID of nodes
+ // that were created without explicitly specifying flags. This also saves time
+ // and allows a gradual increase in API usage of the optional optimization
+ // flags.
+ if (RawFlags != 0)
+ ID.AddInteger(RawFlags);
-/// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
-static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
- bool nuw, bool nsw, bool exact) {
- if (isBinOpWithFlags(Opcode))
- AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
+static void AddNodeIDFlags(FoldingSetNodeID &ID, const SDNode *N) {
+ if (auto *Node = dyn_cast<BinaryWithFlagsSDNode>(N))
+ AddNodeIDFlags(ID, Node->getOpcode(), &Node->Flags);
}
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
}
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
ID.AddInteger(ST->getPointerInfo().getAddrSpace());
break;
}
ID.AddInteger(ST->getPointerInfo().getAddrSpace());
break;
}
- case ISD::SDIV:
- case ISD::UDIV:
- case ISD::SRA:
- case ISD::SRL:
- case ISD::MUL:
- case ISD::ADD:
- case ISD::SUB:
- case ISD::SHL: {
- const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
- AddBinaryNodeIDCustom(
- ID, N->getOpcode(), BinNode->Flags.hasNoUnsignedWrap(),
- BinNode->Flags.hasNoSignedWrap(), BinNode->Flags.hasExact());
- break;
- }
case ISD::ATOMIC_CMP_SWAP:
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
case ISD::ATOMIC_SWAP:
case ISD::ATOMIC_CMP_SWAP:
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
case ISD::ATOMIC_SWAP:
}
} // end switch (N->getOpcode())
}
} // end switch (N->getOpcode())
+ AddNodeIDFlags(ID, N);
+
// Target specific memory nodes could also have address spaces to check.
if (N->isTargetMemoryOpcode())
ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
// Target specific memory nodes could also have address spaces to check.
if (N->isTargetMemoryOpcode())
ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
SDVTList VTs, SDValue N1,
BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
SDVTList VTs, SDValue N1,
- SDValue N2, bool nuw, bool nsw,
- bool exact) {
+ SDValue N2,
+ const SDNodeFlags *Flags) {
if (isBinOpWithFlags(Opcode)) {
if (isBinOpWithFlags(Opcode)) {
+ // If no flags were passed in, use a default flags object.
+ SDNodeFlags F;
+ if (Flags == nullptr)
+ Flags = &F;
+
BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
- Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
- FN->Flags.setNoUnsignedWrap(nuw);
- FN->Flags.setNoSignedWrap(nsw);
- FN->Flags.setExact(exact);
+ Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags);
}
SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
}
SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
- SDValue N2, bool nuw, bool nsw, bool exact) {
+ SDValue N2, const SDNodeFlags *Flags) {
ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
switch (Opcode) {
ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
switch (Opcode) {
// Memoize this node if possible.
BinarySDNode *N;
SDVTList VTs = getVTList(VT);
// Memoize this node if possible.
BinarySDNode *N;
SDVTList VTs = getVTList(VT);
- const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
if (VT != MVT::Glue) {
SDValue Ops[] = {N1, N2};
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTs, Ops);
if (VT != MVT::Glue) {
SDValue Ops[] = {N1, N2};
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTs, Ops);
- if (BinOpHasFlags)
- AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
+ AddNodeIDFlags(ID, Opcode, Flags);
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
- N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
+ N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
CSEMap.InsertNode(N, IP);
} else {
CSEMap.InsertNode(N, IP);
} else {
- N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
+ N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
/// getNodeIfExists - Get the specified node if it's already available, or
/// else return NULL.
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
/// getNodeIfExists - Get the specified node if it's already available, or
/// else return NULL.
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
- ArrayRef<SDValue> Ops, bool nuw, bool nsw,
- bool exact) {
+ ArrayRef<SDValue> Ops,
+ const SDNodeFlags *Flags) {
if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops);
if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops);
- if (isBinOpWithFlags(Opcode))
- AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
+ AddNodeIDFlags(ID, Opcode, Flags);
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DebugLoc(), IP))
return E;
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DebugLoc(), IP))
return E;
cl::location(LimitFloatPrecision),
cl::init(0));
cl::location(LimitFloatPrecision),
cl::init(0));
+static cl::opt<bool>
+EnableFMFInDAG("enable-fmf-dag", cl::init(false), cl::Hidden,
+ cl::desc("Enable fast-math-flags for DAG nodes"));
+
// Limit the width of DAG chains. This is important in general to prevent
// prevent DAG-based analysis from blowing up. For example, alias analysis and
// load clustering may not complete in reasonable time. It is difficult to
// Limit the width of DAG chains. This is important in general to prevent
// prevent DAG-based analysis from blowing up. For example, alias analysis and
// load clustering may not complete in reasonable time. It is difficult to
bool nuw = false;
bool nsw = false;
bool exact = false;
bool nuw = false;
bool nsw = false;
bool exact = false;
if (const OverflowingBinaryOperator *OFBinOp =
dyn_cast<const OverflowingBinaryOperator>(&I)) {
nuw = OFBinOp->hasNoUnsignedWrap();
if (const OverflowingBinaryOperator *OFBinOp =
dyn_cast<const OverflowingBinaryOperator>(&I)) {
nuw = OFBinOp->hasNoUnsignedWrap();
if (const PossiblyExactOperator *ExactOp =
dyn_cast<const PossiblyExactOperator>(&I))
exact = ExactOp->isExact();
if (const PossiblyExactOperator *ExactOp =
dyn_cast<const PossiblyExactOperator>(&I))
exact = ExactOp->isExact();
+ if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
+ FMF = FPOp->getFastMathFlags();
+
+ SDNodeFlags Flags;
+ Flags.setExact(exact);
+ Flags.setNoSignedWrap(nsw);
+ Flags.setNoUnsignedWrap(nuw);
+ if (EnableFMFInDAG) {
+ Flags.setAllowReciprocal(FMF.allowReciprocal());
+ Flags.setNoInfs(FMF.noInfs());
+ Flags.setNoNaNs(FMF.noNaNs());
+ Flags.setNoSignedZeros(FMF.noSignedZeros());
+ Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
+ }
SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
- Op1, Op2, nuw, nsw, exact);
setValue(&I, BinNodeValue);
}
setValue(&I, BinNodeValue);
}
dyn_cast<const PossiblyExactOperator>(&I))
exact = ExactOp->isExact();
}
dyn_cast<const PossiblyExactOperator>(&I))
exact = ExactOp->isExact();
}
+ SDNodeFlags Flags;
+ Flags.setExact(exact);
+ Flags.setNoSignedWrap(nsw);
+ Flags.setNoUnsignedWrap(nuw);
SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
// TODO: For UDIV use SRL instead of SRA.
SDValue Amt =
DAG.getConstant(ShAmt, dl, getShiftAmountTy(Op1.getValueType()));
// TODO: For UDIV use SRL instead of SRA.
SDValue Amt =
DAG.getConstant(ShAmt, dl, getShiftAmountTy(Op1.getValueType()));
- Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, false, false,
- true);
+ SDNodeFlags Flags;
+ Flags.setExact(true);
+ Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, &Flags);