#include "llvm/Instructions.h"
#include "llvm/Intrinsics.h"
#include "llvm/IntrinsicInst.h"
+#include "llvm/Module.h"
#include "llvm/CodeGen/FastISel.h"
#include "llvm/CodeGen/GCStrategy.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/Target/TargetRegisterInfo.h"
#include "llvm/Target/TargetData.h"
ValueVT : MVT::getIntegerVT(RoundBits);
SDValue Lo, Hi;
+ MVT HalfVT = ValueVT.isInteger() ?
+ MVT::getIntegerVT(RoundBits/2) :
+ MVT::getFloatingPointVT(RoundBits/2);
+
if (RoundParts > 2) {
- MVT HalfVT = MVT::getIntegerVT(RoundBits/2);
Lo = getCopyFromParts(DAG, Parts, RoundParts/2, PartVT, HalfVT);
Hi = getCopyFromParts(DAG, Parts+RoundParts/2, RoundParts/2,
PartVT, HalfVT);
} else {
- Lo = Parts[0];
- Hi = Parts[1];
+ Lo = DAG.getNode(ISD::BIT_CONVERT, HalfVT, Parts[0]);
+ Hi = DAG.getNode(ISD::BIT_CONVERT, HalfVT, Parts[1]);
}
if (TLI.isBigEndian())
std::swap(Lo, Hi);
SDValue(TrueVal.getNode(), TrueVal.getResNo() + i),
SDValue(FalseVal.getNode(), FalseVal.getResNo() + i));
- setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], NumValues),
- &Values[0], NumValues));
+ setValue(&I, DAG.getNode(ISD::MERGE_VALUES,
+ DAG.getVTList(&ValueVTs[0], NumValues),
+ &Values[0], NumValues));
}
}
TLI.getValueType(I.getType()), InVec, InIdx));
}
+
+// Utility for visitShuffleVector - Returns true if the mask is mask starting
+// from SIndx and increasing to the element length (undefs are allowed).
+static bool SequentialMask(SDValue Mask, unsigned SIndx) {
+ unsigned MaskNumElts = Mask.getNumOperands();
+ for (unsigned i = 0; i != MaskNumElts; ++i) {
+ if (Mask.getOperand(i).getOpcode() != ISD::UNDEF) {
+ unsigned Idx = cast<ConstantSDNode>(Mask.getOperand(i))->getZExtValue();
+ if (Idx != i + SIndx)
+ return false;
+ }
+ }
+ return true;
+}
+
void SelectionDAGLowering::visitShuffleVector(User &I) {
- SDValue V1 = getValue(I.getOperand(0));
- SDValue V2 = getValue(I.getOperand(1));
+ SDValue Src1 = getValue(I.getOperand(0));
+ SDValue Src2 = getValue(I.getOperand(1));
SDValue Mask = getValue(I.getOperand(2));
- setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE,
- TLI.getValueType(I.getType()),
- V1, V2, Mask));
+ MVT VT = TLI.getValueType(I.getType());
+ MVT SrcVT = Src1.getValueType();
+ int MaskNumElts = Mask.getNumOperands();
+ int SrcNumElts = SrcVT.getVectorNumElements();
+
+ if (SrcNumElts == MaskNumElts) {
+ setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Src1, Src2, Mask));
+ return;
+ }
+
+ // Normalize the shuffle vector since mask and vector length don't match.
+ MVT MaskEltVT = Mask.getValueType().getVectorElementType();
+
+ if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
+ // Mask is longer than the source vectors and is a multiple of the source
+ // vectors. We can use concatenate vector to make the mask and vectors
+ // lengths match.
+ if (SrcNumElts*2 == MaskNumElts && SequentialMask(Mask, 0)) {
+ // The shuffle is concatenating two vectors together.
+ setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, VT, Src1, Src2));
+ return;
+ }
+
+ // Pad both vectors with undefs to make them the same length as the mask.
+ unsigned NumConcat = MaskNumElts / SrcNumElts;
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, SrcVT);
+
+ SDValue* MOps1 = new SDValue[NumConcat];
+ SDValue* MOps2 = new SDValue[NumConcat];
+ MOps1[0] = Src1;
+ MOps2[0] = Src2;
+ for (unsigned i = 1; i != NumConcat; ++i) {
+ MOps1[i] = UndefVal;
+ MOps2[i] = UndefVal;
+ }
+ Src1 = DAG.getNode(ISD::CONCAT_VECTORS, VT, MOps1, NumConcat);
+ Src2 = DAG.getNode(ISD::CONCAT_VECTORS, VT, MOps2, NumConcat);
+
+ delete [] MOps1;
+ delete [] MOps2;
+
+ // Readjust mask for new input vector length.
+ SmallVector<SDValue, 8> MappedOps;
+ for (int i = 0; i != MaskNumElts; ++i) {
+ if (Mask.getOperand(i).getOpcode() == ISD::UNDEF) {
+ MappedOps.push_back(Mask.getOperand(i));
+ } else {
+ int Idx = cast<ConstantSDNode>(Mask.getOperand(i))->getZExtValue();
+ if (Idx < SrcNumElts)
+ MappedOps.push_back(DAG.getConstant(Idx, MaskEltVT));
+ else
+ MappedOps.push_back(DAG.getConstant(Idx + MaskNumElts - SrcNumElts,
+ MaskEltVT));
+ }
+ }
+ Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(),
+ &MappedOps[0], MappedOps.size());
+
+ setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Src1, Src2, Mask));
+ return;
+ }
+
+ if (SrcNumElts > MaskNumElts) {
+ // Resulting vector is shorter than the incoming vector.
+ if (SrcNumElts == MaskNumElts && SequentialMask(Mask,0)) {
+ // Shuffle extracts 1st vector.
+ setValue(&I, Src1);
+ return;
+ }
+
+ if (SrcNumElts == MaskNumElts && SequentialMask(Mask,MaskNumElts)) {
+ // Shuffle extracts 2nd vector.
+ setValue(&I, Src2);
+ return;
+ }
+
+ // Analyze the access pattern of the vector to see if we can extract
+ // two subvectors and do the shuffle. The analysis is done by calculating
+ // the range of elements the mask access on both vectors.
+ int MinRange[2] = { SrcNumElts+1, SrcNumElts+1};
+ int MaxRange[2] = {-1, -1};
+
+ for (int i = 0; i != MaskNumElts; ++i) {
+ SDValue Arg = Mask.getOperand(i);
+ if (Arg.getOpcode() != ISD::UNDEF) {
+ assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
+ int Idx = cast<ConstantSDNode>(Arg)->getZExtValue();
+ int Input = 0;
+ if (Idx >= SrcNumElts) {
+ Input = 1;
+ Idx -= SrcNumElts;
+ }
+ if (Idx > MaxRange[Input])
+ MaxRange[Input] = Idx;
+ if (Idx < MinRange[Input])
+ MinRange[Input] = Idx;
+ }
+ }
+
+ // Check if the access is smaller than the vector size and can we find
+ // a reasonable extract index.
+ int RangeUse[2] = { 2, 2 }; // 0 = Unused, 1 = Extract, 2 = Can not Extract.
+ int StartIdx[2]; // StartIdx to extract from
+ for (int Input=0; Input < 2; ++Input) {
+ if (MinRange[Input] == SrcNumElts+1 && MaxRange[Input] == -1) {
+ RangeUse[Input] = 0; // Unused
+ StartIdx[Input] = 0;
+ } else if (MaxRange[Input] - MinRange[Input] < MaskNumElts) {
+ // Fits within range but we should see if we can find a good
+ // start index that is a multiple of the mask length.
+ if (MaxRange[Input] < MaskNumElts) {
+ RangeUse[Input] = 1; // Extract from beginning of the vector
+ StartIdx[Input] = 0;
+ } else {
+ StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
+ if (MaxRange[Input] - StartIdx[Input] < MaskNumElts &&
+ StartIdx[Input] + MaskNumElts < SrcNumElts)
+ RangeUse[Input] = 1; // Extract from a multiple of the mask length.
+ }
+ }
+ }
+
+ if (RangeUse[0] == 0 && RangeUse[0] == 0) {
+ setValue(&I, DAG.getNode(ISD::UNDEF, VT)); // Vectors are not used.
+ return;
+ }
+ else if (RangeUse[0] < 2 && RangeUse[1] < 2) {
+ // Extract appropriate subvector and generate a vector shuffle
+ for (int Input=0; Input < 2; ++Input) {
+ SDValue& Src = Input == 0 ? Src1 : Src2;
+ if (RangeUse[Input] == 0) {
+ Src = DAG.getNode(ISD::UNDEF, VT);
+ } else {
+ Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, VT, Src,
+ DAG.getIntPtrConstant(StartIdx[Input]));
+ }
+ }
+ // Calculate new mask.
+ SmallVector<SDValue, 8> MappedOps;
+ for (int i = 0; i != MaskNumElts; ++i) {
+ SDValue Arg = Mask.getOperand(i);
+ if (Arg.getOpcode() == ISD::UNDEF) {
+ MappedOps.push_back(Arg);
+ } else {
+ int Idx = cast<ConstantSDNode>(Arg)->getZExtValue();
+ if (Idx < SrcNumElts)
+ MappedOps.push_back(DAG.getConstant(Idx - StartIdx[0], MaskEltVT));
+ else {
+ Idx = Idx - SrcNumElts - StartIdx[1] + MaskNumElts;
+ MappedOps.push_back(DAG.getConstant(Idx, MaskEltVT));
+ }
+ }
+ }
+ Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(),
+ &MappedOps[0], MappedOps.size());
+ setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Src1, Src2, Mask));
+ return;
+ }
+ }
+
+ // We can't use either concat vectors or extract subvectors so fall back to
+ // replacing the shuffle with extract and build vector.
+ // to insert and build vector.
+ MVT EltVT = VT.getVectorElementType();
+ MVT PtrVT = TLI.getPointerTy();
+ SmallVector<SDValue,8> Ops;
+ for (int i = 0; i != MaskNumElts; ++i) {
+ SDValue Arg = Mask.getOperand(i);
+ if (Arg.getOpcode() == ISD::UNDEF) {
+ Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT));
+ } else {
+ assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
+ int Idx = cast<ConstantSDNode>(Arg)->getZExtValue();
+ if (Idx < SrcNumElts)
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Src1,
+ DAG.getConstant(Idx, PtrVT)));
+ else
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Src2,
+ DAG.getConstant(Idx - SrcNumElts, PtrVT)));
+ }
+ }
+ setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()));
}
void SelectionDAGLowering::visitInsertValue(InsertValueInst &I) {
Values[i] = IntoUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) :
SDValue(Agg.getNode(), Agg.getResNo() + i);
- setValue(&I, DAG.getMergeValues(DAG.getVTList(&AggValueVTs[0], NumAggValues),
- &Values[0], NumAggValues));
+ setValue(&I, DAG.getNode(ISD::MERGE_VALUES,
+ DAG.getVTList(&AggValueVTs[0], NumAggValues),
+ &Values[0], NumAggValues));
}
void SelectionDAGLowering::visitExtractValue(ExtractValueInst &I) {
// Copy out the selected value(s).
for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
Values[i - LinearIndex] =
- OutOfUndef ? DAG.getNode(ISD::UNDEF, Agg.getNode()->getValueType(Agg.getResNo() + i)) :
- SDValue(Agg.getNode(), Agg.getResNo() + i);
-
- setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValValueVTs[0], NumValValues),
- &Values[0], NumValValues));
+ OutOfUndef ?
+ DAG.getNode(ISD::UNDEF,
+ Agg.getNode()->getValueType(Agg.getResNo() + i)) :
+ SDValue(Agg.getNode(), Agg.getResNo() + i);
+
+ setValue(&I, DAG.getNode(ISD::MERGE_VALUES,
+ DAG.getVTList(&ValValueVTs[0], NumValValues),
+ &Values[0], NumValValues));
}
PendingLoads.push_back(Chain);
}
- setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], NumValues),
- &Values[0], NumValues));
+ setValue(&I, DAG.getNode(ISD::MERGE_VALUES,
+ DAG.getVTList(&ValueVTs[0], NumValues),
+ &Values[0], NumValues));
}
Ops.push_back(getRoot());
}
}
-
- // Add the intrinsic ID as an integer operand.
- Ops.push_back(DAG.getConstant(Intrinsic, TLI.getPointerTy()));
+
+ // Info is set by getTgtMemInstrinsic
+ TargetLowering::IntrinsicInfo Info;
+ bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
+
+ // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
+ if (!IsTgtIntrinsic)
+ Ops.push_back(DAG.getConstant(Intrinsic, TLI.getPointerTy()));
// Add all operands of the call to the operand list.
for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) {
// Create the node.
SDValue Result;
- if (!HasChain)
+ if (IsTgtIntrinsic) {
+ // This is target intrinsic that touches memory
+ Result = DAG.getMemIntrinsicNode(Info.opc, VTList, VTs.size(),
+ &Ops[0], Ops.size(),
+ Info.memVT, Info.ptrVal, Info.offset,
+ Info.align, Info.vol,
+ Info.readMem, Info.writeMem);
+ }
+ else if (!HasChain)
Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VTList, VTs.size(),
&Ops[0], Ops.size());
else if (I.getType() != Type::VoidTy)
return 0;
}
+// implVisitAluOverflow - Lower an overflow instrinsics
+const char *
+SelectionDAGLowering::implVisitAluOverflow(CallInst &I, ISD::NodeType Op) {
+ SDValue Op1 = getValue(I.getOperand(1));
+ SDValue Op2 = getValue(I.getOperand(2));
+
+ MVT ValueVTs[] = { Op1.getValueType(), MVT::i1 };
+ SDValue Ops[] = { Op1, Op2 };
+
+ SDValue Result =
+ DAG.getNode(Op,
+ DAG.getVTList(&ValueVTs[0], 2), &Ops[0], 2);
+
+ setValue(&I, Result);
+ return 0;
+ }
+
/// visitExp - Lower an exp intrinsic. Handles the special sequences for
/// limited-precision mode.
void
case Intrinsic::longjmp:
return "_longjmp"+!TLI.usesUnderscoreLongJmp();
break;
- case Intrinsic::memcpy_i32:
- case Intrinsic::memcpy_i64: {
+ case Intrinsic::memcpy: {
SDValue Op1 = getValue(I.getOperand(1));
SDValue Op2 = getValue(I.getOperand(2));
SDValue Op3 = getValue(I.getOperand(3));
I.getOperand(1), 0, I.getOperand(2), 0));
return 0;
}
- case Intrinsic::memset_i32:
- case Intrinsic::memset_i64: {
+ case Intrinsic::memset: {
SDValue Op1 = getValue(I.getOperand(1));
SDValue Op2 = getValue(I.getOperand(2));
SDValue Op3 = getValue(I.getOperand(3));
I.getOperand(1), 0));
return 0;
}
- case Intrinsic::memmove_i32:
- case Intrinsic::memmove_i64: {
+ case Intrinsic::memmove: {
SDValue Op1 = getValue(I.getOperand(1));
SDValue Op2 = getValue(I.getOperand(2));
SDValue Op3 = getValue(I.getOperand(3));
SubprogramDesc *Subprogram = cast<SubprogramDesc>(DD);
const CompileUnitDesc *CompileUnit = Subprogram->getFile();
unsigned SrcFile = MMI->RecordSource(CompileUnit);
- // Record the source line but does create a label. It will be emitted
- // at asm emission time.
- MMI->RecordSourceLine(Subprogram->getLine(), 0, SrcFile);
+ // Record the source line but does not create a label for the normal
+ // function start. It will be emitted at asm emission time. However,
+ // create a label if this is a beginning of inlined function.
+ unsigned LabelID = MMI->RecordSourceLine(Subprogram->getLine(), 0, SrcFile);
+ if (MMI->getSourceLines().size() != 1)
+ DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getRoot(), LabelID));
}
return 0;
return 0;
}
+ case Intrinsic::convertff:
+ case Intrinsic::convertfsi:
+ case Intrinsic::convertfui:
+ case Intrinsic::convertsif:
+ case Intrinsic::convertuif:
+ case Intrinsic::convertss:
+ case Intrinsic::convertsu:
+ case Intrinsic::convertus:
+ case Intrinsic::convertuu: {
+ ISD::CvtCode Code = ISD::CVT_INVALID;
+ switch (Intrinsic) {
+ case Intrinsic::convertff: Code = ISD::CVT_FF; break;
+ case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
+ case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
+ case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
+ case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
+ case Intrinsic::convertss: Code = ISD::CVT_SS; break;
+ case Intrinsic::convertsu: Code = ISD::CVT_SU; break;
+ case Intrinsic::convertus: Code = ISD::CVT_US; break;
+ case Intrinsic::convertuu: Code = ISD::CVT_UU; break;
+ }
+ MVT DestVT = TLI.getValueType(I.getType());
+ Value* Op1 = I.getOperand(1);
+ setValue(&I, DAG.getConvertRndSat(DestVT, getValue(Op1),
+ DAG.getValueType(DestVT),
+ DAG.getValueType(getValue(Op1).getValueType()),
+ getValue(I.getOperand(2)),
+ getValue(I.getOperand(3)),
+ Code));
+ return 0;
+ }
+
case Intrinsic::sqrt:
setValue(&I, DAG.getNode(ISD::FSQRT,
getValue(I.getOperand(1)).getValueType(),
DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, MVT::Other, getRoot(), Tmp));
return 0;
}
+ case Intrinsic::stackprotector: {
+ // Emit code into the DAG to store the stack guard onto the stack.
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineFrameInfo *MFI = MF.getFrameInfo();
+ MVT PtrTy = TLI.getPointerTy();
+
+ SDValue Src = getValue(I.getOperand(1)); // The guard's value.
+ AllocaInst *Slot = cast<AllocaInst>(I.getOperand(2));
+
+ int FI = FuncInfo.StaticAllocaMap[Slot];
+ MFI->setStackProtectorIndex(FI);
+
+ SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
+
+ // Store the stack protector onto the stack.
+ SDValue Result = DAG.getStore(getRoot(), Src, FIN,
+ PseudoSourceValue::getFixedStack(FI),
+ 0, true);
+ setValue(&I, Result);
+ DAG.setRoot(Result);
+ return 0;
+ }
case Intrinsic::var_annotation:
// Discard annotate attributes
return 0;
DAG.setRoot(DAG.getNode(ISD::TRAP, MVT::Other, getRoot()));
return 0;
}
+
+ case Intrinsic::uadd_with_overflow:
+ return implVisitAluOverflow(I, ISD::UADDO);
+ case Intrinsic::sadd_with_overflow:
+ return implVisitAluOverflow(I, ISD::SADDO);
+ case Intrinsic::usub_with_overflow:
+ return implVisitAluOverflow(I, ISD::USUBO);
+ case Intrinsic::ssub_with_overflow:
+ return implVisitAluOverflow(I, ISD::SSUBO);
+ case Intrinsic::umul_with_overflow:
+ return implVisitAluOverflow(I, ISD::UMULO);
+ case Intrinsic::smul_with_overflow:
+ return implVisitAluOverflow(I, ISD::SMULO);
+
case Intrinsic::prefetch: {
SDValue Ops[4];
Ops[0] = getRoot();
Parts.clear();
}
- return DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], ValueVTs.size()),
- &Values[0], ValueVTs.size());
+ return DAG.getNode(ISD::MERGE_VALUES,
+ DAG.getVTList(&ValueVTs[0], ValueVTs.size()),
+ &Values[0], ValueVTs.size());
}
/// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
AssertOp);
ReturnValues.push_back(ReturnValue);
}
- Res = DAG.getMergeValues(DAG.getVTList(&RetTys[0], RetTys.size()),
- &ReturnValues[0], ReturnValues.size());
+ Res = DAG.getNode(ISD::MERGE_VALUES,
+ DAG.getVTList(&RetTys[0], RetTys.size()),
+ &ReturnValues[0], ReturnValues.size());
}
return std::make_pair(Res, Chain);
projects / oota-llvm.git / blobdiff