bool foldXALUIntrinsic(AArch64CC::CondCode &CC, const Instruction *I,
const Value *Cond);
bool optimizeIntExtLoad(const Instruction *I, MVT RetVT, MVT SrcVT);
+ bool optimizeSelect(const SelectInst *SI);
+ std::pair<unsigned, bool> getRegForGEPIndex(const Value *Idx);
// Emit helper routines.
unsigned emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
RetVT.SimpleTy = std::max(RetVT.SimpleTy, MVT::i32);
// Canonicalize immediates to the RHS first.
- if (UseAdd && isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS))
+ if (UseAdd && isa<Constant>(LHS) && !isa<Constant>(RHS))
std::swap(LHS, RHS);
// Canonicalize mul by power of 2 to the RHS.
else
ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, Imm, SetFlags,
WantResult);
- }
+ } else if (const auto *C = dyn_cast<Constant>(RHS))
+ if (C->isNullValue())
+ ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, 0, SetFlags,
+ WantResult);
+
if (ResultReg)
return ResultReg;
const Value *LHS = CI->getOperand(0);
const Value *RHS = CI->getOperand(1);
- Type *Ty = LHS->getType();
- if (!Ty->isIntegerTy())
- return false;
+ MVT VT;
+ if (!isTypeSupported(LHS->getType(), VT))
+ return false;
- unsigned BW = cast<IntegerType>(Ty)->getBitWidth();
- if (BW != 1 && BW != 8 && BW != 16 && BW != 32 && BW != 64)
+ unsigned BW = VT.getSizeInBits();
+ if (BW > 64)
return false;
MachineBasicBlock *TBB = FuncInfo.MBBMap[BI->getSuccessor(0)];
int TestBit = -1;
bool IsCmpNE;
if ((Predicate == CmpInst::ICMP_EQ) || (Predicate == CmpInst::ICMP_NE)) {
- if (const auto *C = dyn_cast<ConstantInt>(LHS))
+ if (const auto *C = dyn_cast<Constant>(LHS))
if (C->isNullValue())
std::swap(LHS, RHS);
- if (!isa<ConstantInt>(RHS))
+ if (!isa<Constant>(RHS))
return false;
- if (!cast<ConstantInt>(RHS)->isNullValue())
+ if (!cast<Constant>(RHS)->isNullValue())
return false;
if (const auto *AI = dyn_cast<BinaryOperator>(LHS))
- if (AI->getOpcode() == Instruction::And) {
+ if (AI->getOpcode() == Instruction::And && isValueAvailable(AI)) {
const Value *AndLHS = AI->getOperand(0);
const Value *AndRHS = AI->getOperand(1);
LHS = AndLHS;
}
}
+
+ if (VT == MVT::i1)
+ TestBit = 0;
+
IsCmpNE = Predicate == CmpInst::ICMP_NE;
} else if (Predicate == CmpInst::ICMP_SLT) {
- if (!isa<ConstantInt>(RHS))
+ if (!isa<Constant>(RHS))
return false;
- if (!cast<ConstantInt>(RHS)->isNullValue())
+ if (!cast<Constant>(RHS)->isNullValue())
return false;
TestBit = BW - 1;
bool Is64Bit = BW == 64;
if (TestBit < 32 && TestBit >= 0)
Is64Bit = false;
-
+
unsigned Opc = OpcTable[IsBitTest][IsCmpNE][Is64Bit];
const MCInstrDesc &II = TII.get(Opc);
SrcReg = fastEmitInst_extractsubreg(MVT::i32, SrcReg, SrcIsKill,
AArch64::sub_32);
- if ((BW < 32) && !IsBitTest) {
- EVT CmpEVT = TLI.getValueType(Ty, true);
- SrcReg =
- emitIntExt(CmpEVT.getSimpleVT(), SrcReg, MVT::i32, /*isZExt*/ true);
- }
+ if ((BW < 32) && !IsBitTest)
+ SrcReg = emitIntExt(VT, SrcReg, MVT::i32, /*IsZExt=*/true);
// Emit the combined compare and branch instruction.
SrcReg = constrainOperandRegClass(II, SrcReg, II.getNumDefs());
return true;
}
-bool AArch64FastISel::selectSelect(const Instruction *I) {
- const SelectInst *SI = cast<SelectInst>(I);
+/// \brief Optimize selects of i1 if one of the operands has a 'true' or 'false'
+/// value.
+bool AArch64FastISel::optimizeSelect(const SelectInst *SI) {
+ if (!SI->getType()->isIntegerTy(1))
+ return false;
- EVT DestEVT = TLI.getValueType(SI->getType(), true);
- if (!DestEVT.isSimple())
+ const Value *Src1Val, *Src2Val;
+ unsigned Opc = 0;
+ bool NeedExtraOp = false;
+ if (auto *CI = dyn_cast<ConstantInt>(SI->getTrueValue())) {
+ if (CI->isOne()) {
+ Src1Val = SI->getCondition();
+ Src2Val = SI->getFalseValue();
+ Opc = AArch64::ORRWrr;
+ } else {
+ assert(CI->isZero());
+ Src1Val = SI->getFalseValue();
+ Src2Val = SI->getCondition();
+ Opc = AArch64::BICWrr;
+ }
+ } else if (auto *CI = dyn_cast<ConstantInt>(SI->getFalseValue())) {
+ if (CI->isOne()) {
+ Src1Val = SI->getCondition();
+ Src2Val = SI->getTrueValue();
+ Opc = AArch64::ORRWrr;
+ NeedExtraOp = true;
+ } else {
+ assert(CI->isZero());
+ Src1Val = SI->getCondition();
+ Src2Val = SI->getTrueValue();
+ Opc = AArch64::ANDWrr;
+ }
+ }
+
+ if (!Opc)
return false;
- MVT DestVT = DestEVT.getSimpleVT();
- if (DestVT != MVT::i32 && DestVT != MVT::i64 && DestVT != MVT::f32 &&
- DestVT != MVT::f64)
+ unsigned Src1Reg = getRegForValue(Src1Val);
+ if (!Src1Reg)
return false;
+ bool Src1IsKill = hasTrivialKill(Src1Val);
- unsigned SelectOpc;
- const TargetRegisterClass *RC = nullptr;
- switch (DestVT.SimpleTy) {
- default: return false;
+ unsigned Src2Reg = getRegForValue(Src2Val);
+ if (!Src2Reg)
+ return false;
+ bool Src2IsKill = hasTrivialKill(Src2Val);
+
+ if (NeedExtraOp) {
+ Src1Reg = emitLogicalOp_ri(ISD::XOR, MVT::i32, Src1Reg, Src1IsKill, 1);
+ Src1IsKill = true;
+ }
+ unsigned ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32spRegClass, Src1Reg,
+ Src1IsKill, Src2Reg, Src2IsKill);
+ updateValueMap(SI, ResultReg);
+ return true;
+}
+
+bool AArch64FastISel::selectSelect(const Instruction *I) {
+ assert(isa<SelectInst>(I) && "Expected a select instruction.");
+ MVT VT;
+ if (!isTypeSupported(I->getType(), VT))
+ return false;
+
+ unsigned Opc;
+ const TargetRegisterClass *RC;
+ switch (VT.SimpleTy) {
+ default:
+ return false;
+ case MVT::i1:
+ case MVT::i8:
+ case MVT::i16:
case MVT::i32:
- SelectOpc = AArch64::CSELWr; RC = &AArch64::GPR32RegClass; break;
+ Opc = AArch64::CSELWr;
+ RC = &AArch64::GPR32RegClass;
+ break;
case MVT::i64:
- SelectOpc = AArch64::CSELXr; RC = &AArch64::GPR64RegClass; break;
+ Opc = AArch64::CSELXr;
+ RC = &AArch64::GPR64RegClass;
+ break;
case MVT::f32:
- SelectOpc = AArch64::FCSELSrrr; RC = &AArch64::FPR32RegClass; break;
+ Opc = AArch64::FCSELSrrr;
+ RC = &AArch64::FPR32RegClass;
+ break;
case MVT::f64:
- SelectOpc = AArch64::FCSELDrrr; RC = &AArch64::FPR64RegClass; break;
+ Opc = AArch64::FCSELDrrr;
+ RC = &AArch64::FPR64RegClass;
+ break;
}
+ const SelectInst *SI = cast<SelectInst>(I);
const Value *Cond = SI->getCondition();
- bool NeedTest = true;
AArch64CC::CondCode CC = AArch64CC::NE;
- if (foldXALUIntrinsic(CC, I, Cond))
- NeedTest = false;
+ AArch64CC::CondCode ExtraCC = AArch64CC::AL;
- unsigned CondReg = getRegForValue(Cond);
- if (!CondReg)
- return false;
- bool CondIsKill = hasTrivialKill(Cond);
+ if (optimizeSelect(SI))
+ return true;
- if (NeedTest) {
- unsigned ANDReg = emitAnd_ri(MVT::i32, CondReg, CondIsKill, 1);
- assert(ANDReg && "Unexpected AND instruction emission failure.");
- emitICmp_ri(MVT::i32, ANDReg, /*IsKill=*/true, 0);
+ // Try to pickup the flags, so we don't have to emit another compare.
+ if (foldXALUIntrinsic(CC, I, Cond)) {
+ // Fake request the condition to force emission of the XALU intrinsic.
+ unsigned CondReg = getRegForValue(Cond);
+ if (!CondReg)
+ return false;
+ } else if (isa<CmpInst>(Cond) && cast<CmpInst>(Cond)->hasOneUse() &&
+ isValueAvailable(Cond)) {
+ const auto *Cmp = cast<CmpInst>(Cond);
+ // Try to optimize or fold the cmp.
+ CmpInst::Predicate Predicate = optimizeCmpPredicate(Cmp);
+ const Value *FoldSelect = nullptr;
+ switch (Predicate) {
+ default:
+ break;
+ case CmpInst::FCMP_FALSE:
+ FoldSelect = SI->getFalseValue();
+ break;
+ case CmpInst::FCMP_TRUE:
+ FoldSelect = SI->getTrueValue();
+ break;
+ }
+
+ if (FoldSelect) {
+ unsigned SrcReg = getRegForValue(FoldSelect);
+ if (!SrcReg)
+ return false;
+ unsigned UseReg = lookUpRegForValue(SI);
+ if (UseReg)
+ MRI.clearKillFlags(UseReg);
+
+ updateValueMap(I, SrcReg);
+ return true;
+ }
+
+ // Emit the cmp.
+ if (!emitCmp(Cmp->getOperand(0), Cmp->getOperand(1), Cmp->isUnsigned()))
+ return false;
+
+ // FCMP_UEQ and FCMP_ONE cannot be checked with a single select instruction.
+ CC = getCompareCC(Predicate);
+ switch (Predicate) {
+ default:
+ break;
+ case CmpInst::FCMP_UEQ:
+ ExtraCC = AArch64CC::EQ;
+ CC = AArch64CC::VS;
+ break;
+ case CmpInst::FCMP_ONE:
+ ExtraCC = AArch64CC::MI;
+ CC = AArch64CC::GT;
+ break;
+ }
+ assert((CC != AArch64CC::AL) && "Unexpected condition code.");
+ } else {
+ unsigned CondReg = getRegForValue(Cond);
+ if (!CondReg)
+ return false;
+ bool CondIsKill = hasTrivialKill(Cond);
+
+ // Emit a TST instruction (ANDS wzr, reg, #imm).
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ANDSWri),
+ AArch64::WZR)
+ .addReg(CondReg, getKillRegState(CondIsKill))
+ .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
}
- unsigned TrueReg = getRegForValue(SI->getTrueValue());
- bool TrueIsKill = hasTrivialKill(SI->getTrueValue());
+ unsigned Src1Reg = getRegForValue(SI->getTrueValue());
+ bool Src1IsKill = hasTrivialKill(SI->getTrueValue());
- unsigned FalseReg = getRegForValue(SI->getFalseValue());
- bool FalseIsKill = hasTrivialKill(SI->getFalseValue());
+ unsigned Src2Reg = getRegForValue(SI->getFalseValue());
+ bool Src2IsKill = hasTrivialKill(SI->getFalseValue());
- if (!TrueReg || !FalseReg)
+ if (!Src1Reg || !Src2Reg)
return false;
- unsigned ResultReg = fastEmitInst_rri(SelectOpc, RC, TrueReg, TrueIsKill,
- FalseReg, FalseIsKill, CC);
+ if (ExtraCC != AArch64CC::AL) {
+ Src2Reg = fastEmitInst_rri(Opc, RC, Src1Reg, Src1IsKill, Src2Reg,
+ Src2IsKill, ExtraCC);
+ Src2IsKill = true;
+ }
+ unsigned ResultReg = fastEmitInst_rri(Opc, RC, Src1Reg, Src1IsKill, Src2Reg,
+ Src2IsKill, CC);
updateValueMap(I, ResultReg);
return true;
}
updateValueMap(II, CLI.ResultReg);
return true;
}
+ case Intrinsic::fabs: {
+ MVT VT;
+ if (!isTypeLegal(II->getType(), VT))
+ return false;
+
+ unsigned Opc;
+ switch (VT.SimpleTy) {
+ default:
+ return false;
+ case MVT::f32:
+ Opc = AArch64::FABSSr;
+ break;
+ case MVT::f64:
+ Opc = AArch64::FABSDr;
+ break;
+ }
+ unsigned SrcReg = getRegForValue(II->getOperand(0));
+ if (!SrcReg)
+ return false;
+ bool SrcRegIsKill = hasTrivialKill(II->getOperand(0));
+ unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
+ .addReg(SrcReg, getKillRegState(SrcRegIsKill));
+ updateValueMap(II, ResultReg);
+ return true;
+ }
case Intrinsic::trap: {
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::BRK))
.addImm(1);
unsigned RegSize = Is64Bit ? 64 : 32;
unsigned DstBits = RetVT.getSizeInBits();
unsigned SrcBits = SrcVT.getSizeInBits();
+ const TargetRegisterClass *RC =
+ Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
+
+ // Just emit a copy for "zero" shifts.
+ if (Shift == 0) {
+ unsigned ResultReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), ResultReg)
+ .addReg(Op0, getKillRegState(Op0IsKill));
+ return ResultReg;
+ }
// Don't deal with undefined shifts.
if (Shift >= DstBits)
{AArch64::UBFMWri, AArch64::UBFMXri}
};
unsigned Opc = OpcTable[IsZext][Is64Bit];
- const TargetRegisterClass *RC =
- Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
unsigned TmpReg = MRI.createVirtualRegister(RC);
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
unsigned RegSize = Is64Bit ? 64 : 32;
unsigned DstBits = RetVT.getSizeInBits();
unsigned SrcBits = SrcVT.getSizeInBits();
+ const TargetRegisterClass *RC =
+ Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
+
+ // Just emit a copy for "zero" shifts.
+ if (Shift == 0) {
+ unsigned ResultReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), ResultReg)
+ .addReg(Op0, getKillRegState(Op0IsKill));
+ return ResultReg;
+ }
// Don't deal with undefined shifts.
if (Shift >= DstBits)
{AArch64::UBFMWri, AArch64::UBFMXri}
};
unsigned Opc = OpcTable[IsZExt][Is64Bit];
- const TargetRegisterClass *RC =
- Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
unsigned TmpReg = MRI.createVirtualRegister(RC);
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
unsigned RegSize = Is64Bit ? 64 : 32;
unsigned DstBits = RetVT.getSizeInBits();
unsigned SrcBits = SrcVT.getSizeInBits();
+ const TargetRegisterClass *RC =
+ Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
+
+ // Just emit a copy for "zero" shifts.
+ if (Shift == 0) {
+ unsigned ResultReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), ResultReg)
+ .addReg(Op0, getKillRegState(Op0IsKill));
+ return ResultReg;
+ }
// Don't deal with undefined shifts.
if (Shift >= DstBits)
{AArch64::UBFMWri, AArch64::UBFMXri}
};
unsigned Opc = OpcTable[IsZExt][Is64Bit];
- const TargetRegisterClass *RC =
- Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
unsigned TmpReg = MRI.createVirtualRegister(RC);
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
.addImm(AArch64::sub_32);
SrcReg = ResultReg;
}
+ // Conservatively clear all kill flags from all uses, because we are
+ // replacing a sign-/zero-extend instruction at IR level with a nop at MI
+ // level. The result of the instruction at IR level might have been
+ // trivially dead, which is now not longer true.
+ unsigned UseReg = lookUpRegForValue(I);
+ if (UseReg)
+ MRI.clearKillFlags(UseReg);
+
updateValueMap(I, SrcReg);
return true;
}
return true;
}
+/// This is mostly a copy of the existing FastISel getRegForGEPIndex code. We
+/// have to duplicate it for AArch64, because otherwise we would fail during the
+/// sign-extend emission.
+std::pair<unsigned, bool> AArch64FastISel::getRegForGEPIndex(const Value *Idx) {
+ unsigned IdxN = getRegForValue(Idx);
+ if (IdxN == 0)
+ // Unhandled operand. Halt "fast" selection and bail.
+ return std::pair<unsigned, bool>(0, false);
+
+ bool IdxNIsKill = hasTrivialKill(Idx);
+
+ // If the index is smaller or larger than intptr_t, truncate or extend it.
+ MVT PtrVT = TLI.getPointerTy();
+ EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
+ if (IdxVT.bitsLT(PtrVT)) {
+ IdxN = emitIntExt(IdxVT.getSimpleVT(), IdxN, PtrVT, /*IsZExt=*/false);
+ IdxNIsKill = true;
+ } else if (IdxVT.bitsGT(PtrVT))
+ llvm_unreachable("AArch64 FastISel doesn't support types larger than i64");
+ return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
+}
+
/// This is mostly a copy of the existing FastISel GEP code, but we have to
/// duplicate it for AArch64, because otherwise we would bail out even for
/// simple cases. This is because the standard fastEmit functions don't cover
projects / oota-llvm.git / blobdiff