///
//===---------------------------------------------------------------------===//
namespace {
- cl::opt<RegisterScheduler::FunctionPassCtor, false,
- RegisterPassParser<RegisterScheduler> >
+ static cl::opt<RegisterScheduler::FunctionPassCtor, false,
+ RegisterPassParser<RegisterScheduler> >
ISHeuristic("pre-RA-sched",
cl::init(&createDefaultScheduler),
cl::desc("Instruction schedulers available (before register"
namespace { struct SDISelAsmOperandInfo; }
-namespace {
- /// ComputeValueVTs - Given an LLVM IR type, compute a sequence of
- /// MVT::ValueTypes that represent all the individual underlying
- /// non-aggregate types that comprise it.
- static void ComputeValueVTs(const TargetLowering &TLI,
- const Type *Ty,
- SmallVectorImpl<MVT::ValueType> &ValueVTs) {
- // Given a struct type, recursively traverse the elements.
- if (const StructType *STy = dyn_cast<StructType>(Ty)) {
- for (StructType::element_iterator EI = STy->element_begin(),
- EB = STy->element_end();
- EI != EB; ++EI)
- ComputeValueVTs(TLI, *EI, ValueVTs);
- return;
- }
- // Given an array type, recursively traverse the elements.
- if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
- const Type *EltTy = ATy->getElementType();
- for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i)
- ComputeValueVTs(TLI, EltTy, ValueVTs);
- return;
- }
- // Base case: we can get an MVT::ValueType for this LLVM IR type.
- MVT::ValueType VT = TLI.getValueType(Ty);
- ValueVTs.push_back(VT);
+/// ComputeValueVTs - Given an LLVM IR type, compute a sequence of
+/// MVT::ValueTypes that represent all the individual underlying
+/// non-aggregate types that comprise it.
+static void ComputeValueVTs(const TargetLowering &TLI, const Type *Ty,
+ SmallVectorImpl<MVT::ValueType> &ValueVTs) {
+ // Given a struct type, recursively traverse the elements.
+ if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+ for (StructType::element_iterator EI = STy->element_begin(),
+ EB = STy->element_end();
+ EI != EB; ++EI)
+ ComputeValueVTs(TLI, *EI, ValueVTs);
+ return;
+ }
+ // Given an array type, recursively traverse the elements.
+ if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+ const Type *EltTy = ATy->getElementType();
+ for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i)
+ ComputeValueVTs(TLI, EltTy, ValueVTs);
+ return;
}
+ // Base case: we can get an MVT::ValueType for this LLVM IR type.
+ ValueVTs.push_back(TLI.getValueType(Ty));
+}
- /// RegsForValue - This struct represents the physical registers that a
- /// particular value is assigned and the type information about the value.
- /// This is needed because values can be promoted into larger registers and
- /// expanded into multiple smaller registers than the value.
+namespace {
+ /// RegsForValue - This struct represents the registers (physical or virtual)
+ /// that a particular set of values is assigned, and the type information about
+ /// the value. The most common situation is to represent one value at a time,
+ /// but struct or array values are handled element-wise as multiple values.
+ /// The splitting of aggregates is performed recursively, so that we never
+ /// have aggregate-typed registers. The values at this point do not necessarily
+ /// have legal types, so each value may require one or more registers of some
+ /// legal type.
+ ///
struct VISIBILITY_HIDDEN RegsForValue {
/// TLI - The TargetLowering object.
+ ///
const TargetLowering *TLI;
- /// Regs - This list holds the register (for legal and promoted values)
- /// or register set (for expanded values) that the value should be assigned
- /// to.
- std::vector<unsigned> Regs;
+ /// ValueVTs - The value types of the values, which may not be legal, and
+ /// may need be promoted or synthesized from one or more registers.
+ ///
+ SmallVector<MVT::ValueType, 4> ValueVTs;
- /// RegVTs - The value types of the registers. This is the same size
- /// as ValueVTs; every register contributing to a given value must
- /// have the same type. When Regs contains all virtual registers, the
- /// contents of RegVTs is redundant with TLI's getRegisterType member
- /// function, however when Regs contains physical registers, it is
- /// necessary to have a separate record of the types.
+ /// RegVTs - The value types of the registers. This is the same size as
+ /// ValueVTs and it records, for each value, what the type of the assigned
+ /// register or registers are. (Individual values are never synthesized
+ /// from more than one type of register.)
+ ///
+ /// With virtual registers, the contents of RegVTs is redundant with TLI's
+ /// getRegisterType member function, however when with physical registers
+ /// it is necessary to have a separate record of the types.
///
SmallVector<MVT::ValueType, 4> RegVTs;
- /// ValueVTs - The value types of the values, which may be promoted
- /// or synthesized from one or more registers.
- SmallVector<MVT::ValueType, 4> ValueVTs;
+ /// Regs - This list holds the registers assigned to the values.
+ /// Each legal or promoted value requires one register, and each
+ /// expanded value requires multiple registers.
+ ///
+ SmallVector<unsigned, 4> Regs;
RegsForValue() : TLI(0) {}
RegsForValue(const TargetLowering &tli,
- unsigned Reg, MVT::ValueType regvt, MVT::ValueType valuevt)
- : TLI(&tli), Regs(1, Reg), RegVTs(1, regvt), ValueVTs(1, valuevt) {}
- RegsForValue(const TargetLowering &tli,
- const std::vector<unsigned> ®s,
+ const SmallVector<unsigned, 4> ®s,
MVT::ValueType regvt, MVT::ValueType valuevt)
- : TLI(&tli), Regs(regs), RegVTs(1, regvt), ValueVTs(1, valuevt) {}
+ : TLI(&tli), ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
RegsForValue(const TargetLowering &tli,
- const std::vector<unsigned> ®s,
+ const SmallVector<unsigned, 4> ®s,
const SmallVector<MVT::ValueType, 4> ®vts,
const SmallVector<MVT::ValueType, 4> &valuevts)
- : TLI(&tli), Regs(regs), RegVTs(regvts), ValueVTs(valuevts) {}
+ : TLI(&tli), ValueVTs(valuevts), RegVTs(regvts), Regs(regs) {}
RegsForValue(const TargetLowering &tli,
unsigned Reg, const Type *Ty) : TLI(&tli) {
ComputeValueVTs(tli, Ty, ValueVTs);
- for (unsigned Value = 0; Value != ValueVTs.size(); ++Value) {
+ for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
MVT::ValueType ValueVT = ValueVTs[Value];
unsigned NumRegs = TLI->getNumRegisters(ValueVT);
MVT::ValueType RegisterVT = TLI->getRegisterType(ValueVT);
}
}
+ /// append - Add the specified values to this one.
+ void append(const RegsForValue &RHS) {
+ TLI = RHS.TLI;
+ ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
+ RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
+ Regs.append(RHS.Regs.begin(), RHS.Regs.end());
+ }
+
+
/// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
/// this value and returns the result as a ValueVTs value. This uses
/// Chain/Flag as the input and updates them for the output Chain/Flag.
/// CreateRegForValue - Allocate the appropriate number of virtual registers of
/// the correctly promoted or expanded types. Assign these registers
/// consecutive vreg numbers and return the first assigned number.
+///
+/// In the case that the given value has struct or array type, this function
+/// will assign registers for each member or element.
+///
unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) {
- const Type *Ty = V->getType();
SmallVector<MVT::ValueType, 4> ValueVTs;
- ComputeValueVTs(TLI, Ty, ValueVTs);
+ ComputeValueVTs(TLI, V->getType(), ValueVTs);
unsigned FirstReg = 0;
- for (unsigned Value = 0; Value != ValueVTs.size(); ++Value) {
+ for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
MVT::ValueType ValueVT = ValueVTs[Value];
- unsigned NumRegs = TLI.getNumRegisters(ValueVT);
MVT::ValueType RegisterVT = TLI.getRegisterType(ValueVT);
+ unsigned NumRegs = TLI.getNumRegisters(ValueVT);
for (unsigned i = 0; i != NumRegs; ++i) {
unsigned R = MakeReg(RegisterVT);
if (!FirstReg) FirstReg = R;
assert(0 && "UserOp2 should not exist at instruction selection time!");
abort();
}
+
+private:
+ inline const char *implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op);
+
};
} // end namespace llvm
SDOperand &N = NodeMap[V];
if (N.Val) return N;
- const Type *VTy = V->getType();
- MVT::ValueType VT = TLI.getValueType(VTy, true);
if (Constant *C = const_cast<Constant*>(dyn_cast<Constant>(V))) {
+ MVT::ValueType VT = TLI.getValueType(V->getType(), true);
+
+ if (ConstantInt *CI = dyn_cast<ConstantInt>(C))
+ return N = DAG.getConstant(CI->getValue(), VT);
+
+ if (GlobalValue *GV = dyn_cast<GlobalValue>(C))
+ return N = DAG.getGlobalAddress(GV, VT);
+
+ if (isa<ConstantPointerNull>(C))
+ return N = DAG.getConstant(0, TLI.getPointerTy());
+
+ if (ConstantFP *CFP = dyn_cast<ConstantFP>(C))
+ return N = DAG.getConstantFP(CFP->getValueAPF(), VT);
+
+ if (isa<UndefValue>(C) && !isa<VectorType>(V->getType()))
+ return N = DAG.getNode(ISD::UNDEF, VT);
+
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
visit(CE->getOpcode(), *CE);
SDOperand N1 = NodeMap[V];
assert(N1.Val && "visit didn't populate the ValueMap!");
return N1;
- } else if (GlobalValue *GV = dyn_cast<GlobalValue>(C)) {
- return N = DAG.getGlobalAddress(GV, VT);
- } else if (isa<ConstantPointerNull>(C)) {
- return N = DAG.getConstant(0, TLI.getPointerTy());
- } else if (isa<UndefValue>(C)) {
- if (!isa<VectorType>(VTy))
- return N = DAG.getNode(ISD::UNDEF, VT);
-
- // Create a BUILD_VECTOR of undef nodes.
- const VectorType *PTy = cast<VectorType>(VTy);
- unsigned NumElements = PTy->getNumElements();
- MVT::ValueType PVT = TLI.getValueType(PTy->getElementType());
-
- SmallVector<SDOperand, 8> Ops;
- Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT));
-
- // Create a VConstant node with generic Vector type.
- MVT::ValueType VT = MVT::getVectorType(PVT, NumElements);
- return N = DAG.getNode(ISD::BUILD_VECTOR, VT,
- &Ops[0], Ops.size());
- } else if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
- return N = DAG.getConstantFP(CFP->getValueAPF(), VT);
- } else if (const VectorType *PTy = dyn_cast<VectorType>(VTy)) {
- unsigned NumElements = PTy->getNumElements();
- MVT::ValueType PVT = TLI.getValueType(PTy->getElementType());
-
- // Now that we know the number and type of the elements, push a
- // Constant or ConstantFP node onto the ops list for each element of
- // the vector constant.
- SmallVector<SDOperand, 8> Ops;
- if (ConstantVector *CP = dyn_cast<ConstantVector>(C)) {
- for (unsigned i = 0; i != NumElements; ++i)
- Ops.push_back(getValue(CP->getOperand(i)));
- } else {
- assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
- SDOperand Op;
- if (MVT::isFloatingPoint(PVT))
- Op = DAG.getConstantFP(0, PVT);
- else
- Op = DAG.getConstant(0, PVT);
- Ops.assign(NumElements, Op);
- }
-
- // Create a BUILD_VECTOR node.
- MVT::ValueType VT = MVT::getVectorType(PVT, NumElements);
- return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0],
- Ops.size());
+ }
+
+ const VectorType *VecTy = cast<VectorType>(V->getType());
+ unsigned NumElements = VecTy->getNumElements();
+
+ // Now that we know the number and type of the elements, get that number of
+ // elements into the Ops array based on what kind of constant it is.
+ SmallVector<SDOperand, 16> Ops;
+ if (ConstantVector *CP = dyn_cast<ConstantVector>(C)) {
+ for (unsigned i = 0; i != NumElements; ++i)
+ Ops.push_back(getValue(CP->getOperand(i)));
} else {
- // Canonicalize all constant ints to be unsigned.
- return N = DAG.getConstant(cast<ConstantInt>(C)->getValue(),VT);
+ assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
+ "Unknown vector constant!");
+ MVT::ValueType EltVT = TLI.getValueType(VecTy->getElementType());
+
+ SDOperand Op;
+ if (isa<UndefValue>(C))
+ Op = DAG.getNode(ISD::UNDEF, EltVT);
+ else if (MVT::isFloatingPoint(EltVT))
+ Op = DAG.getConstantFP(0, EltVT);
+ else
+ Op = DAG.getConstant(0, EltVT);
+ Ops.assign(NumElements, Op);
}
+
+ // Create a BUILD_VECTOR node.
+ return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
}
+ // If this is a static alloca, generate it as the frameindex instead of
+ // computation.
if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
std::map<const AllocaInst*, int>::iterator SI =
- FuncInfo.StaticAllocaMap.find(AI);
+ FuncInfo.StaticAllocaMap.find(AI);
if (SI != FuncInfo.StaticAllocaMap.end())
return DAG.getFrameIndex(SI->second, TLI.getPointerTy());
}
unsigned InReg = FuncInfo.ValueMap[V];
assert(InReg && "Value not in map!");
- RegsForValue RFV(TLI, InReg, VTy);
+ RegsForValue RFV(TLI, InReg, V->getType());
SDOperand Chain = DAG.getEntryNode();
-
return RFV.getCopyFromRegs(DAG, Chain, NULL);
}
DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, getControlRoot()));
return;
}
+
SmallVector<SDOperand, 8> NewValues;
NewValues.push_back(getControlRoot());
for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) {
case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break;
case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break;
case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break;
- case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break;
- case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break;
+ case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break;
+ case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break;
case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break;
case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break;
case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break;
case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break;
case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break;
case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break;
- case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break;
- case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break;
+ case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break;
+ case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break;
case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break;
case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break;
case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break;
/// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
static GlobalVariable *ExtractTypeInfo (Value *V) {
- V = IntrinsicInst::StripPointerCasts(V);
+ V = V->stripPointerCasts();
GlobalVariable *GV = dyn_cast<GlobalVariable>(V);
assert ((GV || isa<ConstantPointerNull>(V)) &&
"TypeInfo must be a global variable or NULL");
}
}
+
+/// Inlined utility function to implement binary input atomic intrinsics for
+// visitIntrinsicCall: I is a call instruction
+// Op is the associated NodeType for I
+const char *
+SelectionDAGLowering::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) {
+ SDOperand Root = getRoot();
+ SDOperand O2 = getValue(I.getOperand(2));
+ SDOperand L = DAG.getAtomic(Op, Root,
+ getValue(I.getOperand(1)),
+ O2, O2.getValueType());
+ setValue(&I, L);
+ DAG.setRoot(L.getValue(1));
+ return 0;
+}
+
/// visitIntrinsicCall - Lower the call to the specified intrinsic function. If
/// we want to emit this as a call to a named external function, return the name
/// otherwise lower it and return null.
return 0;
case Intrinsic::init_trampoline: {
- const Function *F =
- cast<Function>(IntrinsicInst::StripPointerCasts(I.getOperand(2)));
+ const Function *F = cast<Function>(I.getOperand(2)->stripPointerCasts());
SDOperand Ops[6];
Ops[0] = getRoot();
DAG.setRoot(L.getValue(1));
return 0;
}
- case Intrinsic::atomic_las: {
- SDOperand Root = getRoot();
- SDOperand O2 = getValue(I.getOperand(2));
- SDOperand L = DAG.getAtomic(ISD::ATOMIC_LAS, Root,
- getValue(I.getOperand(1)),
- O2, O2.getValueType());
- setValue(&I, L);
- DAG.setRoot(L.getValue(1));
- return 0;
- }
- case Intrinsic::atomic_swap: {
- SDOperand Root = getRoot();
- SDOperand O2 = getValue(I.getOperand(2));
- SDOperand L = DAG.getAtomic(ISD::ATOMIC_SWAP, Root,
- getValue(I.getOperand(1)),
- O2, O2.getValueType());
- setValue(&I, L);
- DAG.setRoot(L.getValue(1));
- return 0;
- }
-
+ case Intrinsic::atomic_las:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LAS);
+ case Intrinsic::atomic_lss:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LSS);
+ case Intrinsic::atomic_load_and:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_AND);
+ case Intrinsic::atomic_load_or:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_OR);
+ case Intrinsic::atomic_load_xor:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_XOR);
+ case Intrinsic::atomic_load_min:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MIN);
+ case Intrinsic::atomic_load_max:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MAX);
+ case Intrinsic::atomic_load_umin:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMIN);
+ case Intrinsic::atomic_load_umax:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMAX);
+ case Intrinsic::atomic_swap:
+ return implVisitBinaryAtomic(I, ISD::ATOMIC_SWAP);
}
}
if (isa<UndefValue>(I.getOperand(0))) {
SDOperand Undef = DAG.getNode(ISD::UNDEF, TLI.getValueType(I.getType()));
setValue(&I, Undef);
- } else {
- SDOperand Call = getValue(I.getOperand(0));
-
- // To add support for individual return values with aggregate types,
- // we'd need a way to take a getresult index and determine which
- // values of the Call SDNode are associated with it.
- assert(TLI.getValueType(I.getType(), true) != MVT::Other &&
- "Individual return values must not be aggregates!");
-
- setValue(&I, SDOperand(Call.Val, I.getIndex()));
+ return;
}
+
+ // To add support for individual return values with aggregate types,
+ // we'd need a way to take a getresult index and determine which
+ // values of the Call SDNode are associated with it.
+ assert(TLI.getValueType(I.getType(), true) != MVT::Other &&
+ "Individual return values must not be aggregates!");
+
+ SDOperand Call = getValue(I.getOperand(0));
+ setValue(&I, SDOperand(Call.Val, I.getIndex()));
}
/// Chain/Flag as the input and updates them for the output Chain/Flag.
/// If the Flag pointer is NULL, no flag is used.
SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
- SDOperand &Chain, SDOperand *Flag)const{
+ SDOperand &Chain,
+ SDOperand *Flag) const {
// Assemble the legal parts into the final values.
SmallVector<SDOperand, 4> Values(ValueVTs.size());
- for (unsigned Value = 0, Part = 0; Value != ValueVTs.size(); ++Value) {
+ SmallVector<SDOperand, 8> Parts;
+ for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
// Copy the legal parts from the registers.
MVT::ValueType ValueVT = ValueVTs[Value];
unsigned NumRegs = TLI->getNumRegisters(ValueVT);
MVT::ValueType RegisterVT = RegVTs[Value];
- SmallVector<SDOperand, 8> Parts(NumRegs);
+ Parts.resize(NumRegs);
for (unsigned i = 0; i != NumRegs; ++i) {
- SDOperand P = Flag ?
- DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT, *Flag) :
- DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT);
- Chain = P.getValue(1);
- if (Flag)
+ SDOperand P;
+ if (Flag == 0)
+ P = DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT);
+ else {
+ P = DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT, *Flag);
*Flag = P.getValue(2);
+ }
+ Chain = P.getValue(1);
Parts[Part+i] = P;
}
ValueVT);
Part += NumRegs;
}
+
+ if (ValueVTs.size() == 1)
+ return Values[0];
+
return DAG.getNode(ISD::MERGE_VALUES,
DAG.getVTList(&ValueVTs[0], ValueVTs.size()),
&Values[0], ValueVTs.size());
// Get the list of the values's legal parts.
unsigned NumRegs = Regs.size();
SmallVector<SDOperand, 8> Parts(NumRegs);
- for (unsigned Value = 0, Part = 0; Value != ValueVTs.size(); ++Value) {
+ for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
MVT::ValueType ValueVT = ValueVTs[Value];
unsigned NumParts = TLI->getNumRegisters(ValueVT);
MVT::ValueType RegisterVT = RegVTs[Value];
// Copy the parts into the registers.
SmallVector<SDOperand, 8> Chains(NumRegs);
for (unsigned i = 0; i != NumRegs; ++i) {
- SDOperand Part = Flag ?
- DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag) :
- DAG.getCopyToReg(Chain, Regs[i], Parts[i]);
- Chains[i] = Part.getValue(0);
- if (Flag)
+ SDOperand Part;
+ if (Flag == 0)
+ Part = DAG.getCopyToReg(Chain, Regs[i], Parts[i]);
+ else {
+ Part = DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag);
*Flag = Part.getValue(1);
+ }
+ Chains[i] = Part.getValue(0);
}
- Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs);
+
+ if (NumRegs == 1 || Flag)
+ // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
+ // flagged to it. That is the CopyToReg nodes and the user are considered
+ // a single scheduling unit. If we create a TokenFactor and return it as
+ // chain, then the TokenFactor is both a predecessor (operand) of the
+ // user as well as a successor (the TF operands are flagged to the user).
+ // c1, f1 = CopyToReg
+ // c2, f2 = CopyToReg
+ // c3 = TokenFactor c1, c2
+ // ...
+ // = op c3, ..., f2
+ Chain = Chains[NumRegs-1];
+ else
+ Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs);
}
/// AddInlineAsmOperands - Add this value to the specified inlineasm node
std::vector<SDOperand> &Ops) const {
MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy();
Ops.push_back(DAG.getTargetConstant(Code | (Regs.size() << 3), IntPtrTy));
- for (unsigned Value = 0, Reg = 0; Value != ValueVTs.size(); ++Value) {
- MVT::ValueType ValueVT = ValueVTs[Value];
- unsigned NumRegs = TLI->getNumRegisters(ValueVT);
+ for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
+ unsigned NumRegs = TLI->getNumRegisters(ValueVTs[Value]);
MVT::ValueType RegisterVT = RegVTs[Value];
- for (unsigned i = 0; i != NumRegs; ++i) {
- SDOperand RegOp = DAG.getRegister(Regs[Reg+i], RegisterVT);
- Ops.push_back(RegOp);
- }
- Reg += NumRegs;
+ for (unsigned i = 0; i != NumRegs; ++i)
+ Ops.push_back(DAG.getRegister(Regs[Reg++], RegisterVT));
}
}
MachineFunction &MF = DAG.getMachineFunction();
- std::vector<unsigned> Regs;
+ SmallVector<unsigned, 4> Regs;
// If this is a constraint for a single physreg, or a constraint for a
// register class, find it.
}
// Otherwise, we couldn't allocate enough registers for this.
- return;
}
bool SawEarlyClobber = false;
unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
+ unsigned ResNo = 0; // ResNo - The result number of the next output.
for (unsigned i = 0, e = ConstraintInfos.size(); i != e; ++i) {
ConstraintOperands.push_back(SDISelAsmOperandInfo(ConstraintInfos[i]));
SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
// Compute the value type for each operand.
switch (OpInfo.Type) {
case InlineAsm::isOutput:
- if (!OpInfo.isIndirect) {
- // The return value of the call is this value. As such, there is no
- // corresponding argument.
- assert(CS.getType() != Type::VoidTy && "Bad inline asm!");
- OpVT = TLI.getValueType(CS.getType());
- } else {
+ // Indirect outputs just consume an argument.
+ if (OpInfo.isIndirect) {
OpInfo.CallOperandVal = CS.getArgument(ArgNo++);
+ break;
+ }
+ // The return value of the call is this value. As such, there is no
+ // corresponding argument.
+ assert(CS.getType() != Type::VoidTy && "Bad inline asm!");
+ if (const StructType *STy = dyn_cast<StructType>(CS.getType())) {
+ OpVT = TLI.getValueType(STy->getElementType(ResNo));
+ } else {
+ assert(ResNo == 0 && "Asm only has one result!");
+ OpVT = TLI.getValueType(CS.getType());
}
+ ++ResNo;
break;
case InlineAsm::isInput:
OpInfo.CallOperandVal = CS.getArgument(ArgNo++);
OpInfo.ConstraintVT = OpVT;
// Compute the constraint code and ConstraintType to use.
- TLI.ComputeConstraintToUse(OpInfo);
+ TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
// Keep track of whether we see an earlyclobber.
SawEarlyClobber |= OpInfo.isEarlyClobber;
// Loop over all of the inputs, copying the operand values into the
// appropriate registers and processing the output regs.
RegsForValue RetValRegs;
-
+
// IndirectStoresToEmit - The set of stores to emit after the inline asm node.
std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit;
exit(1);
}
- if (!OpInfo.isIndirect) {
- // This is the result value of the call.
- assert(RetValRegs.Regs.empty() &&
- "Cannot have multiple output constraints yet!");
- assert(CS.getType() != Type::VoidTy && "Bad inline asm!");
- RetValRegs = OpInfo.AssignedRegs;
- } else {
+ // If this is an indirect operand, store through the pointer after the
+ // asm.
+ if (OpInfo.isIndirect) {
IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
OpInfo.CallOperandVal));
+ } else {
+ // This is the result value of the call.
+ assert(CS.getType() != Type::VoidTy && "Bad inline asm!");
+ // Concatenate this output onto the outputs list.
+ RetValRegs.append(OpInfo.AssignedRegs);
}
// Add information to the INLINEASM node to know that this register is
// Add NumOps>>3 registers to MatchedRegs.
RegsForValue MatchedRegs;
MatchedRegs.TLI = &TLI;
- MatchedRegs.ValueVTs.resize(1, InOperandVal.getValueType());
- MatchedRegs.RegVTs.resize(1, AsmNodeOperands[CurOp+1].getValueType());
+ MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType());
+ MatchedRegs.RegVTs.push_back(AsmNodeOperands[CurOp+1].getValueType());
for (unsigned i = 0, e = NumOps>>3; i != e; ++i) {
unsigned Reg =
cast<RegisterSDNode>(AsmNodeOperands[++CurOp])->getReg();
// and set it as the value of the call.
if (!RetValRegs.Regs.empty()) {
SDOperand Val = RetValRegs.getCopyFromRegs(DAG, Chain, &Flag);
-
- // If the result of the inline asm is a vector, it may have the wrong
- // width/num elts. Make sure to convert it to the right type with
+
+ // If any of the results of the inline asm is a vector, it may have the
+ // wrong width/num elts. This can happen for register classes that can
+ // contain multiple different value types. The preg or vreg allocated may
+ // not have the same VT as was expected. Convert it to the right type with
// bit_convert.
- if (MVT::isVector(Val.getValueType())) {
- const VectorType *VTy = cast<VectorType>(CS.getType());
- MVT::ValueType DesiredVT = TLI.getValueType(VTy);
-
- Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val);
+ if (const StructType *ResSTy = dyn_cast<StructType>(CS.getType())) {
+ for (unsigned i = 0, e = ResSTy->getNumElements(); i != e; ++i) {
+ if (MVT::isVector(Val.Val->getValueType(i)))
+ Val = DAG.getNode(ISD::BIT_CONVERT,
+ TLI.getValueType(ResSTy->getElementType(i)), Val);
+ }
+ } else {
+ if (MVT::isVector(Val.getValueType()))
+ Val = DAG.getNode(ISD::BIT_CONVERT, TLI.getValueType(CS.getType()),
+ Val);
}
-
+
setValue(CS.getInstruction(), Val);
}
AU.setPreservesAll();
}
-
-
bool SelectionDAGISel::runOnFunction(Function &Fn) {
// Get alias analysis for load/store combining.
AA = &getAnalysis<AliasAnalysis>();
return true;
}
-void SelectionDAGLowering::CopyValueToVirtualRegister(Value *V,
- unsigned Reg) {
+void SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, unsigned Reg) {
SDOperand Op = getValue(V);
assert((Op.getOpcode() != ISD::CopyFromReg ||
cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
}
}
+/// IsFixedFrameObjectWithPosOffset - Check if object is a fixed frame object and
+/// whether object offset >= 0.
+static bool
+IsFixedFrameObjectWithPosOffset(MachineFrameInfo * MFI, SDOperand Op) {
+ if (!isa<FrameIndexSDNode>(Op)) return false;
+
+ FrameIndexSDNode * FrameIdxNode = dyn_cast<FrameIndexSDNode>(Op);
+ int FrameIdx = FrameIdxNode->getIndex();
+ return MFI->isFixedObjectIndex(FrameIdx) &&
+ MFI->getObjectOffset(FrameIdx) >= 0;
+}
+
+/// IsPossiblyOverwrittenArgumentOfTailCall - Check if the operand could
+/// possibly be overwritten when lowering the outgoing arguments in a tail
+/// call. Currently the implementation of this call is very conservative and
+/// assumes all arguments sourcing from FORMAL_ARGUMENTS or a CopyFromReg with
+/// virtual registers would be overwritten by direct lowering.
+static bool IsPossiblyOverwrittenArgumentOfTailCall(SDOperand Op,
+ MachineFrameInfo * MFI) {
+ RegisterSDNode * OpReg = NULL;
+ if (Op.getOpcode() == ISD::FORMAL_ARGUMENTS ||
+ (Op.getOpcode()== ISD::CopyFromReg &&
+ (OpReg = dyn_cast<RegisterSDNode>(Op.getOperand(1))) &&
+ (OpReg->getReg() >= TargetRegisterInfo::FirstVirtualRegister)) ||
+ (Op.getOpcode() == ISD::LOAD &&
+ IsFixedFrameObjectWithPosOffset(MFI, Op.getOperand(1))) ||
+ (Op.getOpcode() == ISD::MERGE_VALUES &&
+ Op.getOperand(Op.ResNo).getOpcode() == ISD::LOAD &&
+ IsFixedFrameObjectWithPosOffset(MFI, Op.getOperand(Op.ResNo).
+ getOperand(1))))
+ return true;
+ return false;
+}
+
/// CheckDAGForTailCallsAndFixThem - This Function looks for CALL nodes in the
/// DAG and fixes their tailcall attribute operand.
static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG,
// eligible (no RET or the target rejects) the attribute is fixed to
// false. The TargetLowering::IsEligibleForTailCallOptimization function
// must correctly identify tail call optimizable calls.
- if (isMarkedTailCall &&
- (Ret==NULL ||
- !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG))) {
+ if (!isMarkedTailCall) continue;
+ if (Ret==NULL ||
+ !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG)) {
+ // Not eligible. Mark CALL node as non tail call.
SmallVector<SDOperand, 32> Ops;
unsigned idx=0;
- for(SDNode::op_iterator I =OpCall.Val->op_begin(),
- E=OpCall.Val->op_end(); I!=E; I++, idx++) {
+ for(SDNode::op_iterator I =OpCall.Val->op_begin(),
+ E = OpCall.Val->op_end(); I != E; I++, idx++) {
if (idx!=3)
Ops.push_back(*I);
- else
+ else
Ops.push_back(DAG.getConstant(false, TLI.getPointerTy()));
}
DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size());
+ } else {
+ // Look for tail call clobbered arguments. Emit a series of
+ // copyto/copyfrom virtual register nodes to protect them.
+ SmallVector<SDOperand, 32> Ops;
+ SDOperand Chain = OpCall.getOperand(0), InFlag;
+ unsigned idx=0;
+ for(SDNode::op_iterator I = OpCall.Val->op_begin(),
+ E = OpCall.Val->op_end(); I != E; I++, idx++) {
+ SDOperand Arg = *I;
+ if (idx > 4 && (idx % 2)) {
+ bool isByVal = cast<ARG_FLAGSSDNode>(OpCall.getOperand(idx+1))->
+ getArgFlags().isByVal();
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineFrameInfo *MFI = MF.getFrameInfo();
+ if (!isByVal &&
+ IsPossiblyOverwrittenArgumentOfTailCall(Arg, MFI)) {
+ MVT::ValueType VT = Arg.getValueType();
+ unsigned VReg = MF.getRegInfo().
+ createVirtualRegister(TLI.getRegClassFor(VT));
+ Chain = DAG.getCopyToReg(Chain, VReg, Arg, InFlag);
+ InFlag = Chain.getValue(1);
+ Arg = DAG.getCopyFromReg(Chain, VReg, VT, InFlag);
+ Chain = Arg.getValue(1);
+ InFlag = Arg.getValue(2);
+ }
+ }
+ Ops.push_back(Arg);
+ }
+ // Link in chain of CopyTo/CopyFromReg.
+ Ops[0] = Chain;
+ DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size());
}
}
}
projects / oota-llvm.git / blobdiff