#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/StackMaps.h"
+#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
const DbgValueInst *DI = DDI.getDI();
DebugLoc dl = DDI.getdl();
unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
- MDNode *Variable = DI->getVariable();
- MDNode *Expr = DI->getExpression();
+ MDLocalVariable *Variable = DI->getVariable();
+ MDExpression *Expr = DI->getExpression();
+ assert(Variable->isValidLocationForIntrinsic(dl) &&
+ "Expected inlined-at fields to agree");
uint64_t Offset = DI->getOffset();
// A dbg.value for an alloca is always indirect.
bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
SDDbgValue *SDV;
if (Val.getNode()) {
- if (!EmitFuncArgumentDbgValue(V, Variable, Expr, Offset, IsIndirect,
+ if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, IsIndirect,
Val)) {
SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(),
IsIndirect, Offset, dl, DbgSDNodeOrder);
// Update machine-CFG edges.
MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
- // Figure out which block is immediately after the current one.
- MachineBasicBlock *NextBlock = nullptr;
- MachineFunction::iterator BBI = BrMBB;
- if (++BBI != FuncInfo.MF->end())
- NextBlock = BBI;
-
if (I.isUnconditional()) {
// Update machine-CFG edges.
BrMBB->addSuccessor(Succ0MBB);
// If this is not a fall-through branch or optimizations are switched off,
// emit the branch.
- if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
+ if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
MVT::Other, getControlRoot(),
DAG.getBasicBlock(Succ0MBB)));
assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
- const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
+ const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
SDValue CmpOp = getValue(CB.CmpMHS);
EVT VT = CmpOp.getValueType();
if (CB.TrueBB != CB.FalseBB)
addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
- // Set NextBlock to be the MBB immediately after the current one, if any.
- // This is used to avoid emitting unnecessary branches to the next block.
- MachineBasicBlock *NextBlock = nullptr;
- MachineFunction::iterator BBI = SwitchBB;
- if (++BBI != FuncInfo.MF->end())
- NextBlock = BBI;
-
// If the lhs block is the next block, invert the condition so that we can
// fall through to the lhs instead of the rhs block.
- if (CB.TrueBB == NextBlock) {
+ if (CB.TrueBB == NextBlock(SwitchBB)) {
std::swap(CB.TrueBB, CB.FalseBB);
SDValue True = DAG.getConstant(1, Cond.getValueType());
Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
Sub.getValueType()),
Sub, DAG.getConstant(JTH.Last - JTH.First, VT), ISD::SETUGT);
- // Set NextBlock to be the MBB immediately after the current one, if any.
- // This is used to avoid emitting unnecessary branches to the next block.
- MachineBasicBlock *NextBlock = nullptr;
- MachineFunction::iterator BBI = SwitchBB;
-
- if (++BBI != FuncInfo.MF->end())
- NextBlock = BBI;
-
SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
MVT::Other, CopyTo, CMP,
DAG.getBasicBlock(JT.Default));
- if (JT.MBB != NextBlock)
+ // Avoid emitting unnecessary branches to the next block.
+ if (JT.MBB != NextBlock(SwitchBB))
BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
DAG.getBasicBlock(JT.MBB));
SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
B.Reg, Sub);
- // Set NextBlock to be the MBB immediately after the current one, if any.
- // This is used to avoid emitting unnecessary branches to the next block.
- MachineBasicBlock *NextBlock = nullptr;
- MachineFunction::iterator BBI = SwitchBB;
- if (++BBI != FuncInfo.MF->end())
- NextBlock = BBI;
-
MachineBasicBlock* MBB = B.Cases[0].ThisBB;
addSuccessorWithWeight(SwitchBB, B.Default);
MVT::Other, CopyTo, RangeCmp,
DAG.getBasicBlock(B.Default));
- if (MBB != NextBlock)
+ // Avoid emitting unnecessary branches to the next block.
+ if (MBB != NextBlock(SwitchBB))
BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
DAG.getBasicBlock(MBB));
MVT::Other, getControlRoot(),
Cmp, DAG.getBasicBlock(B.TargetBB));
- // Set NextBlock to be the MBB immediately after the current one, if any.
- // This is used to avoid emitting unnecessary branches to the next block.
- MachineBasicBlock *NextBlock = nullptr;
- MachineFunction::iterator BBI = SwitchBB;
- if (++BBI != FuncInfo.MF->end())
- NextBlock = BBI;
-
- if (NextMBB != NextBlock)
+ // Avoid emitting unnecessary branches to the next block.
+ if (NextMBB != NextBlock(SwitchBB))
BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
DAG.getBasicBlock(NextMBB));
MachineFunction *CurMF = FuncInfo.MF;
// Figure out which block is immediately after the current one.
- MachineBasicBlock *NextBlock = nullptr;
+ MachineBasicBlock *NextMBB = nullptr;
MachineFunction::iterator BBI = CR.CaseBB;
-
if (++BBI != FuncInfo.MF->end())
- NextBlock = BBI;
+ NextMBB = BBI;
BranchProbabilityInfo *BPI = FuncInfo.BPI;
// If any two of the cases has the same destination, and if one value
Case &Big = *(CR.Range.second-1);
if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
- const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
- const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
+ const APInt& SmallValue = Small.Low->getValue();
+ const APInt& BigValue = Big.Low->getValue();
// Check that there is only one bit different.
if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
}
// Rearrange the case blocks so that the last one falls through if possible.
Case &BackCase = *(CR.Range.second-1);
- if (Size > 1 &&
- NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
- // The last case block won't fall through into 'NextBlock' if we emit the
+ if (Size > 1 && NextMBB && Default != NextMBB && BackCase.BB != NextMBB) {
+ // The last case block won't fall through into 'NextMBB' if we emit the
// branches in this order. See if rearranging a case value would help.
// We start at the bottom as it's the case with the least weight.
for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
- if (I->BB == NextBlock) {
+ if (I->BB == NextMBB) {
std::swap(*I, BackCase);
break;
}
Case& FrontCase = *CR.Range.first;
Case& BackCase = *(CR.Range.second-1);
- const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
- const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue();
+ const APInt &First = FrontCase.Low->getValue();
+ const APInt &Last = BackCase.High->getValue();
APInt TSize(First.getBitWidth(), 0);
for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
std::vector<MachineBasicBlock*> DestBBs;
APInt TEI = First;
for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
- const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
- const APInt &High = cast<ConstantInt>(I->High)->getValue();
+ const APInt &Low = I->Low->getValue();
+ const APInt &High = I->High->getValue();
if (Low.sle(TEI) && TEI.sle(High)) {
DestBBs.push_back(I->BB);
// Calculate weight for each unique destination in CR.
DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
- if (FuncInfo.BPI)
- for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
- DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
- DestWeights.find(I->BB);
- if (Itr != DestWeights.end())
- Itr->second += I->ExtraWeight;
- else
- DestWeights[I->BB] = I->ExtraWeight;
- }
+ if (FuncInfo.BPI) {
+ for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
+ DestWeights[I->BB] += I->ExtraWeight;
+ }
// Update successor info. Add one edge to each unique successor.
BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
- for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
- E = DestBBs.end(); I != E; ++I) {
- if (!SuccsHandled[(*I)->getNumber()]) {
- SuccsHandled[(*I)->getNumber()] = true;
- DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
- DestWeights.find(*I);
- addSuccessorWithWeight(JumpTableBB, *I,
- Itr != DestWeights.end() ? Itr->second : 0);
+ for (MachineBasicBlock *DestBB : DestBBs) {
+ if (!SuccsHandled[DestBB->getNumber()]) {
+ SuccsHandled[DestBB->getNumber()] = true;
+ auto I = DestWeights.find(DestBB);
+ addSuccessorWithWeight(JumpTableBB, DestBB,
+ I != DestWeights.end() ? I->second : 0);
}
}
// Size is the number of Cases represented by this range.
unsigned Size = CR.Range.second - CR.Range.first;
- const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
- const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue();
+ const APInt &First = FrontCase.Low->getValue();
+ const APInt &Last = BackCase.High->getValue();
double FMetric = 0;
CaseItr Pivot = CR.Range.first + Size/2;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
J!=E; ++I, ++J) {
- const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
- const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
+ const APInt &LEnd = I->High->getValue();
+ const APInt &RBegin = J->Low->getValue();
APInt Range = ComputeRange(LEnd, RBegin);
assert((Range - 2ULL).isNonNegative() &&
"Invalid case distance");
CaseRange LHSR(CR.Range.first, Pivot);
CaseRange RHSR(Pivot, CR.Range.second);
- const Constant *C = Pivot->Low;
+ const ConstantInt *C = Pivot->Low;
MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
// We know that we branch to the LHS if the Value being switched on is
// Pivot's Value, then we can branch directly to the LHS's Target,
// rather than creating a leaf node for it.
if ((LHSR.second - LHSR.first) == 1 && LHSR.first->High == CR.GE &&
- cast<ConstantInt>(C)->getValue() ==
- (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
+ C->getValue() == (CR.GE->getValue() + 1LL)) {
TrueBB = LHSR.first->BB;
} else {
TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
// is CR.LT - 1, then we can branch directly to the target block for
// the current Case Value, rather than emitting a RHS leaf node for it.
if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
- cast<ConstantInt>(RHSR.first->Low)->getValue() ==
- (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
+ RHSR.first->Low->getValue() == (CR.LT->getValue() - 1LL)) {
FalseBB = RHSR.first->BB;
} else {
FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
<< "Total number of comparisons: " << numCmps << '\n');
// Compute span of values.
- const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
- const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
+ const APInt& minValue = FrontCase.Low->getValue();
+ const APInt& maxValue = BackCase.High->getValue();
APInt cmpRange = maxValue - minValue;
DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
count++;
}
- const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
- const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
+ const APInt& lowValue = I->Low->getValue();
+ const APInt& highValue = I->High->getValue();
uint64_t lo = (lowValue - lowBound).getZExtValue();
uint64_t hi = (highValue - lowBound).getZExtValue();
return true;
}
-/// Clusterify - Transform simple list of Cases into list of CaseRange's
-void SelectionDAGBuilder::Clusterify(CaseVector& Cases,
- const SwitchInst& SI) {
+void SelectionDAGBuilder::Clusterify(CaseVector &Cases, const SwitchInst *SI) {
BranchProbabilityInfo *BPI = FuncInfo.BPI;
- // Start with "simple" cases.
- for (SwitchInst::ConstCaseIt i : SI.cases()) {
- const BasicBlock *SuccBB = i.getCaseSuccessor();
- MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
-
- uint32_t ExtraWeight =
- BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
-
- Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
- SMBB, ExtraWeight));
- }
- std::sort(Cases.begin(), Cases.end(), CaseCmp());
-
- // Merge case into clusters
- if (Cases.size() >= 2)
- // Must recompute end() each iteration because it may be
- // invalidated by erase if we hold on to it
- for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
- J != Cases.end(); ) {
- const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
- const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
- MachineBasicBlock* nextBB = J->BB;
- MachineBasicBlock* currentBB = I->BB;
-
- // If the two neighboring cases go to the same destination, merge them
- // into a single case.
- if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
- I->High = J->High;
- I->ExtraWeight += J->ExtraWeight;
- J = Cases.erase(J);
- } else {
- I = J++;
- }
+
+ // Extract cases from the switch and sort them.
+ typedef std::pair<const ConstantInt*, unsigned> CasePair;
+ std::vector<CasePair> Sorted;
+ Sorted.reserve(SI->getNumCases());
+ for (auto I : SI->cases())
+ Sorted.push_back(std::make_pair(I.getCaseValue(), I.getSuccessorIndex()));
+ std::sort(Sorted.begin(), Sorted.end(), [](CasePair a, CasePair b) {
+ return a.first->getValue().slt(b.first->getValue());
+ });
+
+ // Merge adjacent cases with the same destination, build Cases vector.
+ assert(Cases.empty() && "Cases should be empty before Clusterify;");
+ Cases.reserve(SI->getNumCases());
+ MachineBasicBlock *PreviousSucc = nullptr;
+ for (CasePair &CP : Sorted) {
+ const ConstantInt *CaseVal = CP.first;
+ unsigned SuccIndex = CP.second;
+ MachineBasicBlock *Succ = FuncInfo.MBBMap[SI->getSuccessor(SuccIndex)];
+ uint32_t Weight = BPI ? BPI->getEdgeWeight(SI->getParent(), SuccIndex) : 0;
+
+ if (PreviousSucc == Succ &&
+ (CaseVal->getValue() - Cases.back().High->getValue()) == 1) {
+ // If this case has the same successor and is a neighbour, merge it into
+ // the previous cluster.
+ Cases.back().High = CaseVal;
+ Cases.back().ExtraWeight += Weight;
+ } else {
+ Cases.push_back(Case(CaseVal, CaseVal, Succ, Weight));
}
+ PreviousSucc = Succ;
+ }
+
DEBUG({
size_t numCmps = 0;
for (auto &I : Cases)
void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
- // Figure out which block is immediately after the current one.
- MachineBasicBlock *NextBlock = nullptr;
- if (SwitchMBB + 1 != FuncInfo.MF->end())
- NextBlock = SwitchMBB + 1;
-
-
// Create a vector of Cases, sorted so that we can efficiently create a binary
// search tree from them.
CaseVector Cases;
- Clusterify(Cases, SI);
+ Clusterify(Cases, &SI);
// Get the default destination MBB.
MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
SwitchMBB->addSuccessor(Default);
// If this is not a fall-through branch, emit the branch.
- if (Default != NextBlock) {
+ if (Default != NextBlock(SwitchMBB)) {
DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
getControlRoot(), DAG.getBasicBlock(Default)));
}
/// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
/// argument, create the corresponding DBG_VALUE machine instruction for it now.
/// At the end of instruction selection, they will be inserted to the entry BB.
-bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V,
- MDNode *Variable,
- MDNode *Expr, int64_t Offset,
- bool IsIndirect,
- const SDValue &N) {
+bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
+ const Value *V, MDLocalVariable *Variable, MDExpression *Expr,
+ MDLocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) {
const Argument *Arg = dyn_cast<Argument>(V);
if (!Arg)
return false;
const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
// Ignore inlined function arguments here.
- DIVariable DV(Variable);
- if (DV.isInlinedFnArgument(MF.getFunction()))
+ //
+ // FIXME: Should we be checking DL->inlinedAt() to determine this?
+ if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
return false;
Optional<MachineOperand> Op;
if (!Op)
return false;
+ assert(Variable->isValidLocationForIntrinsic(DL) &&
+ "Expected inlined-at fields to agree");
if (Op->isReg())
FuncInfo.ArgDbgValues.push_back(
- BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE),
- IsIndirect, Op->getReg(), Offset, Variable, Expr));
+ BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
+ Op->getReg(), Offset, Variable, Expr));
else
FuncInfo.ArgDbgValues.push_back(
- BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
+ BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
.addOperand(*Op)
.addImm(Offset)
.addMetadata(Variable)
if (!Align)
Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
- DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
- MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1))));
+ bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
+ SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
+ false, isTC,
+ MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)));
+ updateDAGForMaybeTailCall(MC);
return nullptr;
}
case Intrinsic::memset: {
if (!Align)
Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
- DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
- MachinePointerInfo(I.getArgOperand(0))));
+ bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
+ SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
+ isTC, MachinePointerInfo(I.getArgOperand(0)));
+ updateDAGForMaybeTailCall(MS);
return nullptr;
}
case Intrinsic::memmove: {
if (!Align)
Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
- DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
- MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1))));
+ bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
+ SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
+ isTC, MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)));
+ updateDAGForMaybeTailCall(MM);
return nullptr;
}
case Intrinsic::dbg_declare: {
const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
- MDNode *Variable = DI.getVariable();
- MDNode *Expression = DI.getExpression();
+ MDLocalVariable *Variable = DI.getVariable();
+ MDExpression *Expression = DI.getExpression();
const Value *Address = DI.getAddress();
- DIVariable DIVar(Variable);
- assert((!DIVar || DIVar.isVariable()) &&
- "Variable in DbgDeclareInst should be either null or a DIVariable.");
- if (!Address || !DIVar) {
+ assert(Variable && "Missing variable");
+ if (!Address) {
DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
return nullptr;
}
if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
Address = BCI->getOperand(0);
// Parameters are handled specially.
- bool isParameter =
- (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
- isa<Argument>(Address));
+ bool isParameter = Variable->getTag() == dwarf::DW_TAG_arg_variable ||
+ isa<Argument>(Address);
const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
else {
// Address is an argument, so try to emit its dbg value using
// virtual register info from the FuncInfo.ValueMap.
- EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false, N);
+ EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
+ N);
return nullptr;
}
} else if (AI)
} else {
// If Address is an argument then try to emit its dbg value using
// virtual register info from the FuncInfo.ValueMap.
- if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false,
+ if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
N)) {
// If variable is pinned by a alloca in dominating bb then
// use StaticAllocaMap.
}
case Intrinsic::dbg_value: {
const DbgValueInst &DI = cast<DbgValueInst>(I);
- DIVariable DIVar(DI.getVariable());
- assert((!DIVar || DIVar.isVariable()) &&
- "Variable in DbgValueInst should be either null or a DIVariable.");
- if (!DIVar)
- return nullptr;
+ assert(DI.getVariable() && "Missing variable");
- MDNode *Variable = DI.getVariable();
- MDNode *Expression = DI.getExpression();
+ MDLocalVariable *Variable = DI.getVariable();
+ MDExpression *Expression = DI.getExpression();
uint64_t Offset = DI.getOffset();
const Value *V = DI.getValue();
if (!V)
if (N.getNode()) {
// A dbg.value for an alloca is always indirect.
bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
- if (!EmitFuncArgumentDbgValue(V, Variable, Expression, Offset,
+ if (!EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset,
IsIndirect, N)) {
SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
IsIndirect, Offset, dl, SDNodeOrder);
setValue(&I, Res);
return nullptr;
}
- case Intrinsic::x86_avx2_vinserti128: {
- EVT DestVT = TLI.getValueType(I.getType());
- EVT ElVT = TLI.getValueType(I.getArgOperand(1)->getType());
- uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
- ElVT.getVectorNumElements();
- Res =
- DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
- getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
- DAG.getConstant(Idx, TLI.getVectorIdxTy()));
- setValue(&I, Res);
- return nullptr;
- }
- case Intrinsic::x86_avx_vextractf128_pd_256:
- case Intrinsic::x86_avx_vextractf128_ps_256:
- case Intrinsic::x86_avx_vextractf128_si_256:
- case Intrinsic::x86_avx2_vextracti128: {
- EVT DestVT = TLI.getValueType(I.getType());
- uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
- DestVT.getVectorNumElements();
- Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
- getValue(I.getArgOperand(0)),
- DAG.getConstant(Idx, TLI.getVectorIdxTy()));
- setValue(&I, Res);
- return nullptr;
- }
case Intrinsic::convertff:
case Intrinsic::convertfsi:
case Intrinsic::convertfui:
}
case Intrinsic::clear_cache:
return TLI.getClearCacheBuiltinName();
+ case Intrinsic::eh_actions:
+ setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
+ return nullptr;
case Intrinsic::donothing:
// ignore
return nullptr;
// Directly emit some FRAME_ALLOC machine instrs. Label assignment emission
// is the same on all targets.
for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
- AllocaInst *Slot =
- cast<AllocaInst>(I.getArgOperand(Idx)->stripPointerCasts());
+ Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
+ if (isa<ConstantPointerNull>(Arg))
+ continue; // Skip null pointers. They represent a hole in index space.
+ AllocaInst *Slot = cast<AllocaInst>(Arg);
assert(FuncInfo.StaticAllocaMap.count(Slot) &&
"can only escape static allocas");
int FI = FuncInfo.StaticAllocaMap[Slot];
MCSymbol *FrameAllocSym =
- MF.getMMI().getContext().getOrCreateFrameAllocSymbol(MF.getName(),
- Idx);
+ MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
+ GlobalValue::getRealLinkageName(MF.getName()), Idx);
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
TII->get(TargetOpcode::FRAME_ALLOC))
.addSym(FrameAllocSym)
auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX));
MCSymbol *FrameAllocSym =
- MF.getMMI().getContext().getOrCreateFrameAllocSymbol(Fn->getName(),
- IdxVal);
+ MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
+ GlobalValue::getRealLinkageName(Fn->getName()), IdxVal);
// Create a TargetExternalSymbol for the label to avoid any target lowering
// that would make this PC relative.
// Skip the first return-type Attribute to get to params.
Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
Args.push_back(Entry);
+
+ // If we have an explicit sret argument that is an Instruction, (i.e., it
+ // might point to function-local memory), we can't meaningfully tail-call.
+ if (Entry.isSRet && isa<Instruction>(V))
+ isTailCall = false;
}
// Check if target-independent constraints permit a tail call here.
"Unexpected number of operands");
// Add information to the INLINEASM node to know about this input.
// See InlineAsm.h isUseOperandTiedToDef.
+ OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
OpInfo.getMatchedOperand());
AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlag,
CallingConv::ID CC = CS.getCallingConv();
bool IsAnyRegCC = CC == CallingConv::AnyReg;
bool HasDef = !CS->getType()->isVoidTy();
- SDValue Callee = getValue(CS->getOperand(2)); // <target>
+ SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
+
+ // Handle immediate and symbolic callees.
+ if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
+ Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(),
+ /*isTarget=*/true);
+ else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
+ Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
+ SDLoc(SymbolicCallee),
+ SymbolicCallee->getValueType(0));
// Get the real number of arguments participating in the call <numArgs>
SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
Ops.push_back(DAG.getTargetConstant(
cast<ConstantSDNode>(NBytesVal)->getZExtValue(), MVT::i32));
- // Assume that the Callee is a constant address.
- // FIXME: handle function symbols in the future.
- Ops.push_back(
- DAG.getIntPtrConstant(cast<ConstantSDNode>(Callee)->getZExtValue(),
- /*isTarget=*/true));
+ // Add the callee.
+ Ops.push_back(Callee);
// Adjust <numArgs> to account for any arguments that have been passed on the
// stack instead.
Entry.Alignment = Align;
CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
+
+ // sret demotion isn't compatible with tail-calls, since the sret argument
+ // points into the callers stack frame.
+ CLI.IsTailCall = false;
} else {
for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
EVT VT = RetTys[I];
SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
- // Check successor nodes' PHI nodes that expect a constant to be available
- // from this block.
+ // Check PHI nodes in successors that expect a value to be available from this
+ // block.
for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
const BasicBlock *SuccBB = TI->getSuccessor(succ);
if (!isa<PHINode>(SuccBB->begin())) continue;
SuccMBB, BranchProbabilityInfo::getBranchWeightStackProtector(IsLikely));
return SuccMBB;
}
+
+MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
+ MachineFunction::iterator I = MBB;
+ if (++I == FuncInfo.MF->end())
+ return nullptr;
+ return I;
+}
+
+/// During lowering new call nodes can be created (such as memset, etc.).
+/// Those will become new roots of the current DAG, but complications arise
+/// when they are tail calls. In such cases, the call lowering will update
+/// the root, but the builder still needs to know that a tail call has been
+/// lowered in order to avoid generating an additional return.
+void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
+ // If the node is null, we do have a tail call.
+ if (MaybeTC.getNode() != nullptr)
+ DAG.setRoot(MaybeTC);
+ else
+ HasTailCall = true;
+}
+