setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Custom);
setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Custom);
+
+ setTargetDAGCombine(ISD::MUL);
}
//===----------------------------------------------------------------------===//
Chains.push_back(LowerConstantInitializer(Init->getAggregateElement(i),
GV, Ptr, Chain, DAG));
}
- return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, &Chains[0],
- Chains.size());
+ return DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
+ Chains.data(), Chains.size());
} else {
Init->dump();
llvm_unreachable("Unhandled constant initializer");
for (unsigned i = 1; i < (*I)->getNumOperands(); ++i) {
Ops.push_back((*I)->getOperand(i));
}
- DAG.UpdateNodeOperands(*I, &Ops[0], Ops.size());
+ DAG.UpdateNodeOperands(*I, Ops.data(), Ops.size());
}
return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op),
getPointerTy(AMDGPUAS::CONSTANT_ADDRESS));
}
}
-void AMDGPUTargetLowering::ExtractVectorElements(SDValue Op, SelectionDAG &DAG,
- SmallVectorImpl<SDValue> &Args,
- unsigned Start,
- unsigned Count) const {
- EVT VT = Op.getValueType();
- for (unsigned i = Start, e = Start + Count; i != e; ++i) {
- Args.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op),
- VT.getVectorElementType(),
- Op, DAG.getConstant(i, MVT::i32)));
- }
-}
-
SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
SelectionDAG &DAG) const {
SmallVector<SDValue, 8> Args;
SDValue A = Op.getOperand(0);
SDValue B = Op.getOperand(1);
- ExtractVectorElements(A, DAG, Args, 0,
- A.getValueType().getVectorNumElements());
- ExtractVectorElements(B, DAG, Args, 0,
- B.getValueType().getVectorNumElements());
+ DAG.ExtractVectorElements(A, Args);
+ DAG.ExtractVectorElements(B, Args);
return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(),
- &Args[0], Args.size());
+ Args.data(), Args.size());
}
SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
SelectionDAG &DAG) const {
SmallVector<SDValue, 8> Args;
- EVT VT = Op.getValueType();
unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
- ExtractVectorElements(Op.getOperand(0), DAG, Args, Start,
- VT.getVectorNumElements());
+ EVT VT = Op.getValueType();
+ DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
+ VT.getVectorNumElements());
return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(),
- &Args[0], Args.size());
+ Args.data(), Args.size());
}
SDValue AMDGPUTargetLowering::LowerFrameIndex(SDValue Op,
case AMDGPUIntrinsic::AMDGPU_umin:
return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1),
Op.getOperand(2));
+
+ case AMDGPUIntrinsic::AMDGPU_bfe_i32:
+ return DAG.getNode(AMDGPUISD::BFE_I32, DL, VT,
+ Op.getOperand(1),
+ Op.getOperand(2),
+ Op.getOperand(3));
+
+ case AMDGPUIntrinsic::AMDGPU_bfe_u32:
+ return DAG.getNode(AMDGPUISD::BFE_U32, DL, VT,
+ Op.getOperand(1),
+ Op.getOperand(2),
+ Op.getOperand(3));
+
+ case AMDGPUIntrinsic::AMDGPU_bfi:
+ return DAG.getNode(AMDGPUISD::BFI, DL, VT,
+ Op.getOperand(1),
+ Op.getOperand(2),
+ Op.getOperand(3));
+
+ case AMDGPUIntrinsic::AMDGPU_bfm:
+ return DAG.getNode(AMDGPUISD::BFM, DL, VT,
+ Op.getOperand(1),
+ Op.getOperand(2));
+
case AMDGPUIntrinsic::AMDIL_round_nearest:
return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1));
}
MemEltVT, Store->isVolatile(), Store->isNonTemporal(),
Store->getAlignment()));
}
- return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, &Chains[0], NumElts);
+ return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains.data(), NumElts);
}
SDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
// Lower loads constant address space global variable loads
if (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
- isa<GlobalVariable>(GetUnderlyingObject(Load->getPointerInfo().V))) {
+ isa<GlobalVariable>(
+ GetUnderlyingObject(Load->getMemOperand()->getValue()))) {
SDValue Ptr = DAG.getZExtOrTrunc(Load->getBasePtr(), DL,
getPointerTy(AMDGPUAS::PRIVATE_ADDRESS));
// Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
Remainder_A_Den, Rem, ISD::SETEQ);
- SDValue Ops[2];
- Ops[0] = Div;
- Ops[1] = Rem;
+ SDValue Ops[2] = {
+ Div,
+ Rem
+ };
return DAG.getMergeValues(Ops, 2, DL);
}
// TODO: Don't scalarize on Evergreen?
unsigned NElts = VT.getVectorNumElements();
SmallVector<SDValue, 8> Args;
- ExtractVectorElements(Src, DAG, Args, 0, NElts);
+ DAG.ExtractVectorElements(Src, Args);
SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
for (unsigned I = 0; I < NElts; ++I)
return ExpandSIGN_EXTEND_INREG(Op, BitsDiff, DAG);
}
+//===----------------------------------------------------------------------===//
+// Custom DAG optimizations
+//===----------------------------------------------------------------------===//
+
+static bool isU24(SDValue Op, SelectionDAG &DAG) {
+ APInt KnownZero, KnownOne;
+ EVT VT = Op.getValueType();
+ DAG.ComputeMaskedBits(Op, KnownZero, KnownOne);
+
+ return (VT.getSizeInBits() - KnownZero.countLeadingOnes()) <= 24;
+}
+
+static bool isI24(SDValue Op, SelectionDAG &DAG) {
+ EVT VT = Op.getValueType();
+
+ // In order for this to be a signed 24-bit value, bit 23, must
+ // be a sign bit.
+ return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
+ // as unsigned 24-bit values.
+ (VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24;
+}
+
+static void simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) {
+
+ SelectionDAG &DAG = DCI.DAG;
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ EVT VT = Op.getValueType();
+
+ APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24);
+ APInt KnownZero, KnownOne;
+ TargetLowering::TargetLoweringOpt TLO(DAG, true, true);
+ if (TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
+ DCI.CommitTargetLoweringOpt(TLO);
+}
+
+SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ SDLoc DL(N);
+
+ switch(N->getOpcode()) {
+ default: break;
+ case ISD::MUL: {
+ EVT VT = N->getValueType(0);
+ SDValue N0 = N->getOperand(0);
+ SDValue N1 = N->getOperand(1);
+ SDValue Mul;
+
+ // FIXME: Add support for 24-bit multiply with 64-bit output on SI.
+ if (VT.isVector() || VT.getSizeInBits() > 32)
+ break;
+
+ if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
+ N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
+ N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
+ Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1);
+ } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
+ N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
+ N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
+ Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1);
+ } else {
+ break;
+ }
+
+ SDValue Reg = DAG.getSExtOrTrunc(Mul, DL, VT);
+
+ return Reg;
+ }
+ case AMDGPUISD::MUL_I24:
+ case AMDGPUISD::MUL_U24: {
+ SDValue N0 = N->getOperand(0);
+ SDValue N1 = N->getOperand(1);
+ simplifyI24(N0, DCI);
+ simplifyI24(N1, DCI);
+ return SDValue();
+ }
+ }
+ return SDValue();
+}
+
//===----------------------------------------------------------------------===//
// Helper functions
//===----------------------------------------------------------------------===//
NODE_NAME_CASE(BFE_I32)
NODE_NAME_CASE(BFI)
NODE_NAME_CASE(BFM)
+ NODE_NAME_CASE(MUL_U24)
+ NODE_NAME_CASE(MUL_I24)
NODE_NAME_CASE(URECIP)
NODE_NAME_CASE(DOT4)
NODE_NAME_CASE(EXPORT)
}
}
+static void computeMaskedBitsForMinMax(const SDValue Op0,
+ const SDValue Op1,
+ APInt &KnownZero,
+ APInt &KnownOne,
+ const SelectionDAG &DAG,
+ unsigned Depth) {
+ APInt Op0Zero, Op0One;
+ APInt Op1Zero, Op1One;
+ DAG.ComputeMaskedBits(Op0, Op0Zero, Op0One, Depth);
+ DAG.ComputeMaskedBits(Op1, Op1Zero, Op1One, Depth);
+
+ KnownZero = Op0Zero & Op1Zero;
+ KnownOne = Op0One & Op1One;
+}
+
void AMDGPUTargetLowering::computeMaskedBitsForTargetNode(
const SDValue Op,
APInt &KnownZero,
APInt &KnownOne,
const SelectionDAG &DAG,
unsigned Depth) const {
+
KnownZero = KnownOne = APInt(KnownOne.getBitWidth(), 0); // Don't know anything.
+ unsigned Opc = Op.getOpcode();
+ switch (Opc) {
+ case ISD::INTRINSIC_WO_CHAIN: {
+ // FIXME: The intrinsic should just use the node.
+ switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) {
+ case AMDGPUIntrinsic::AMDGPU_imax:
+ case AMDGPUIntrinsic::AMDGPU_umax:
+ case AMDGPUIntrinsic::AMDGPU_imin:
+ case AMDGPUIntrinsic::AMDGPU_umin:
+ computeMaskedBitsForMinMax(Op.getOperand(1), Op.getOperand(2),
+ KnownZero, KnownOne, DAG, Depth);
+ break;
+ default:
+ break;
+ }
+
+ break;
+ }
+ case AMDGPUISD::SMAX:
+ case AMDGPUISD::UMAX:
+ case AMDGPUISD::SMIN:
+ case AMDGPUISD::UMIN:
+ computeMaskedBitsForMinMax(Op.getOperand(0), Op.getOperand(1),
+ KnownZero, KnownOne, DAG, Depth);
+ break;
+ default:
+ break;
+ }
}
projects / oota-llvm.git / blobdiff