}
bool runOnMachineFunction(MachineFunction &MF) override {
- ForCodeSize =
- MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize) ||
- MF.getFunction()->hasFnAttribute(Attribute::MinSize);
+ ForCodeSize = MF.getFunction()->optForSize();
Subtarget = &MF.getSubtarget<AArch64Subtarget>();
return SelectionDAGISel::runOnMachineFunction(MF);
}
bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
return SelectShiftedRegister(N, true, Reg, Shift);
}
+ bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
+ return SelectAddrModeIndexed7S(N, 1, Base, OffImm);
+ }
+ bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
+ return SelectAddrModeIndexed7S(N, 2, Base, OffImm);
+ }
+ bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
+ return SelectAddrModeIndexed7S(N, 4, Base, OffImm);
+ }
+ bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
+ return SelectAddrModeIndexed7S(N, 8, Base, OffImm);
+ }
+ bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
+ return SelectAddrModeIndexed7S(N, 16, Base, OffImm);
+ }
bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
return SelectAddrModeIndexed(N, 1, Base, OffImm);
}
SDNode *SelectBitfieldInsertOp(SDNode *N);
SDNode *SelectLIBM(SDNode *N);
+ SDNode *SelectFPConvertWithRound(SDNode *N);
+
+ SDNode *SelectReadRegister(SDNode *N);
+ SDNode *SelectWriteRegister(SDNode *N);
// Include the pieces autogenerated from the target description.
#include "AArch64GenDAGISel.inc"
private:
bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
SDValue &Shift);
+ bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
+ SDValue &OffImm);
bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
SDValue &OffImm);
bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
}
bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
+
+ SDNode *GenerateInexactFlagIfNeeded(const SDValue &In, unsigned InTyVariant,
+ SDLoc DL);
};
} // end anonymous namespace
return true;
}
+/// SelectAddrModeIndexed7S - Select a "register plus scaled signed 7-bit
+/// immediate" address. The "Size" argument is the size in bytes of the memory
+/// reference, which determines the scale.
+bool AArch64DAGToDAGISel::SelectAddrModeIndexed7S(SDValue N, unsigned Size,
+ SDValue &Base,
+ SDValue &OffImm) {
+ SDLoc dl(N);
+ // Base only. The address will be materialized into a register before
+ // the memory is accessed.
+ // add x0, Xbase, #offset
+ // stp x1, x2, [x0]
+ Base = N;
+ OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
+ return true;
+}
+
/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
/// immediate" address. The "Size" argument is the size in bytes of the memory
/// reference, which determines the scale.
bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
SDValue &Base, SDValue &OffImm) {
SDLoc dl(N);
+ const DataLayout &DL = CurDAG->getDataLayout();
const TargetLowering *TLI = getTargetLowering();
if (N.getOpcode() == ISD::FrameIndex) {
int FI = cast<FrameIndexSDNode>(N)->getIndex();
- Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+ Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
return true;
}
const GlobalValue *GV = GAN->getGlobal();
unsigned Alignment = GV->getAlignment();
- const DataLayout *DL = TLI->getDataLayout();
Type *Ty = GV->getType()->getElementType();
if (Alignment == 0 && Ty->isSized())
- Alignment = DL->getABITypeAlignment(Ty);
+ Alignment = DL.getABITypeAlignment(Ty);
if (Alignment >= Size)
return true;
Base = N.getOperand(0);
if (Base.getOpcode() == ISD::FrameIndex) {
int FI = cast<FrameIndexSDNode>(Base)->getIndex();
- Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+ Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
}
OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
return true;
if (Base.getOpcode() == ISD::FrameIndex) {
int FI = cast<FrameIndexSDNode>(Base)->getIndex();
const TargetLowering *TLI = getTargetLowering();
- Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+ Base = CurDAG->getTargetFrameIndex(
+ FI, TLI->getPointerTy(CurDAG->getDataLayout()));
}
OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64);
return true;
// it into an i64.
DstVT = MVT::i32;
}
+ } else if (VT == MVT::f16) {
+ Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
} else if (VT == MVT::f32) {
Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
} else if (VT == MVT::f64 || VT.is64BitVector()) {
}
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
- unsigned &LSB, unsigned &MSB,
+ unsigned &Immr, unsigned &Imms,
bool BiggerPattern) {
assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
"N must be a SHR/SRA operation to call this function");
"Type checking must have been done before calling this function");
// Check for AND + SRL doing several bits extract.
- if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, LSB, MSB))
+ if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, Immr, Imms))
return true;
// we're looking for a shift of a shift
assert(Srl_imm > 0 && Srl_imm < VT.getSizeInBits() &&
"bad amount in shift node!");
- // Note: The width operand is encoded as width-1.
- unsigned Width = VT.getSizeInBits() - Trunc_bits - Srl_imm - 1;
- int sLSB = Srl_imm - Shl_imm;
- if (sLSB < 0)
- return false;
- LSB = sLSB;
- MSB = LSB + Width;
+ int immr = Srl_imm - Shl_imm;
+ Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
+ Imms = VT.getSizeInBits() - Shl_imm - Trunc_bits - 1;
// SRA requires a signed extraction
if (VT == MVT::i32)
Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
}
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
- SDValue &Opd0, unsigned &LSB, unsigned &MSB,
+ SDValue &Opd0, unsigned &Immr, unsigned &Imms,
unsigned NumberOfIgnoredLowBits = 0,
bool BiggerPattern = false) {
if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64)
return false;
break;
case ISD::AND:
- return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, LSB, MSB,
+ return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, Immr, Imms,
NumberOfIgnoredLowBits, BiggerPattern);
case ISD::SRL:
case ISD::SRA:
- return isBitfieldExtractOpFromShr(N, Opc, Opd0, LSB, MSB, BiggerPattern);
+ return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
}
unsigned NOpc = N->getMachineOpcode();
case AArch64::UBFMXri:
Opc = NOpc;
Opd0 = N->getOperand(0);
- LSB = cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
- MSB = cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
+ Immr = cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
+ Imms = cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
return true;
}
// Unreachable
}
SDNode *AArch64DAGToDAGISel::SelectBitfieldExtractOp(SDNode *N) {
- unsigned Opc, LSB, MSB;
+ unsigned Opc, Immr, Imms;
SDValue Opd0;
- if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, LSB, MSB))
+ if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
return nullptr;
EVT VT = N->getValueType(0);
// If the bit extract operation is 64bit but the original type is 32bit, we
// need to add one EXTRACT_SUBREG.
if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
- SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(LSB, dl, MVT::i64),
- CurDAG->getTargetConstant(MSB, dl, MVT::i64)};
+ SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
+ CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
return Node;
}
- SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(LSB, dl, VT),
- CurDAG->getTargetConstant(MSB, dl, VT)};
+ SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
+ CurDAG->getTargetConstant(Imms, dl, VT)};
return CurDAG->SelectNodeTo(N, Opc, VT, Ops);
}
return CurDAG->SelectNodeTo(N, Opc, VT, Ops);
}
+/// GenerateInexactFlagIfNeeded - Insert FRINTX instruction to generate inexact
+/// signal on round-to-integer operations if needed. C11 leaves it
+/// implementation-defined whether these operations trigger an inexact
+/// exception. IEEE says they don't. Unfortunately, Darwin decided they do so
+/// we sometimes have to insert a special instruction just to set the right bit
+/// in FPSR.
+SDNode *AArch64DAGToDAGISel::GenerateInexactFlagIfNeeded(const SDValue &In,
+ unsigned InTyVariant,
+ SDLoc DL) {
+ if (Subtarget->isTargetDarwin() && !TM.Options.UnsafeFPMath) {
+ // Pick the right FRINTX using InTyVariant needed to set the flags.
+ // InTyVariant is 0 for 32-bit and 1 for 64-bit.
+ unsigned FRINTXOpcs[] = { AArch64::FRINTXSr, AArch64::FRINTXDr };
+ return CurDAG->getMachineNode(FRINTXOpcs[InTyVariant], DL,
+ In.getValueType(), MVT::Glue, In);
+ }
+ return nullptr;
+}
+
SDNode *AArch64DAGToDAGISel::SelectLIBM(SDNode *N) {
EVT VT = N->getValueType(0);
unsigned Variant;
unsigned Opc;
- unsigned FRINTXOpcs[] = { AArch64::FRINTXSr, AArch64::FRINTXDr };
if (VT == MVT::f32) {
Variant = 0;
} else
return nullptr; // Unrecognized argument type. Fall back on default codegen.
- // Pick the FRINTX variant needed to set the flags.
- unsigned FRINTXOpc = FRINTXOpcs[Variant];
-
switch (N->getOpcode()) {
default:
return nullptr; // Unrecognized libm ISD node. Fall back on default codegen.
SmallVector<SDValue, 2> Ops;
Ops.push_back(In);
- if (!TM.Options.UnsafeFPMath) {
- SDNode *FRINTX = CurDAG->getMachineNode(FRINTXOpc, dl, VT, MVT::Glue, In);
- Ops.push_back(SDValue(FRINTX, 1));
- }
+ if (SDNode *FRINTXNode = GenerateInexactFlagIfNeeded(In, Variant, dl))
+ Ops.push_back(SDValue(FRINTXNode, 1));
return CurDAG->getMachineNode(Opc, dl, VT, Ops);
}
+/// SelectFPConvertWithRound - Try to combine FP rounding and
+/// FP-INT conversion.
+SDNode *AArch64DAGToDAGISel::SelectFPConvertWithRound(SDNode *N) {
+ SDNode *Op0 = N->getOperand(0).getNode();
+
+ // Return if the round op is used by other nodes, as this would result in two
+ // FRINTX, one each for round and convert.
+ if (!Op0->hasOneUse())
+ return nullptr;
+
+ unsigned InTyVariant;
+ EVT InTy = Op0->getValueType(0);
+ if (InTy == MVT::f32)
+ InTyVariant = 0;
+ else if (InTy == MVT::f64)
+ InTyVariant = 1;
+ else
+ return nullptr;
+
+ unsigned OutTyVariant;
+ EVT OutTy = N->getValueType(0);
+ if (OutTy == MVT::i32)
+ OutTyVariant = 0;
+ else if (OutTy == MVT::i64)
+ OutTyVariant = 1;
+ else
+ return nullptr;
+
+ assert((N->getOpcode() == ISD::FP_TO_SINT
+ || N->getOpcode() == ISD::FP_TO_UINT) && "Unexpected opcode!");
+ unsigned FpConVariant = N->getOpcode() == ISD::FP_TO_SINT ? 0 : 1;
+
+ unsigned Opc;
+ switch (Op0->getOpcode()) {
+ default:
+ return nullptr;
+ case ISD::FCEIL: {
+ unsigned FCVTPOpcs[2][2][2] = {
+ { { AArch64::FCVTPSUWSr, AArch64::FCVTPSUXSr },
+ { AArch64::FCVTPSUWDr, AArch64::FCVTPSUXDr } },
+ { { AArch64::FCVTPUUWSr, AArch64::FCVTPUUXSr },
+ { AArch64::FCVTPUUWDr, AArch64::FCVTPUUXDr } } };
+ Opc = FCVTPOpcs[FpConVariant][InTyVariant][OutTyVariant];
+ break;
+ }
+ case ISD::FFLOOR: {
+ unsigned FCVTMOpcs[2][2][2] = {
+ { { AArch64::FCVTMSUWSr, AArch64::FCVTMSUXSr },
+ { AArch64::FCVTMSUWDr, AArch64::FCVTMSUXDr } },
+ { { AArch64::FCVTMUUWSr, AArch64::FCVTMUUXSr },
+ { AArch64::FCVTMUUWDr, AArch64::FCVTMUUXDr } } };
+ Opc = FCVTMOpcs[FpConVariant][InTyVariant][OutTyVariant];
+ break;
+ }
+ case ISD::FTRUNC: {
+ unsigned FCVTZOpcs[2][2][2] = {
+ { { AArch64::FCVTZSUWSr, AArch64::FCVTZSUXSr },
+ { AArch64::FCVTZSUWDr, AArch64::FCVTZSUXDr } },
+ { { AArch64::FCVTZUUWSr, AArch64::FCVTZUUXSr },
+ { AArch64::FCVTZUUWDr, AArch64::FCVTZUUXDr } } };
+ Opc = FCVTZOpcs[FpConVariant][InTyVariant][OutTyVariant];
+ break;
+ }
+ case ISD::FROUND: {
+ unsigned FCVTAOpcs[2][2][2] = {
+ { { AArch64::FCVTASUWSr, AArch64::FCVTASUXSr },
+ { AArch64::FCVTASUWDr, AArch64::FCVTASUXDr } },
+ { { AArch64::FCVTAUUWSr, AArch64::FCVTAUUXSr },
+ { AArch64::FCVTAUUWDr, AArch64::FCVTAUUXDr } } };
+ Opc = FCVTAOpcs[FpConVariant][InTyVariant][OutTyVariant];
+ break;
+ }
+ }
+
+ SDLoc DL(N);
+ SDValue In = Op0->getOperand(0);
+ SmallVector<SDValue, 2> Ops;
+ Ops.push_back(In);
+
+ if (SDNode *FRINTXNode = GenerateInexactFlagIfNeeded(In, InTyVariant, DL))
+ Ops.push_back(SDValue(FRINTXNode, 1));
+
+ return CurDAG->getMachineNode(Opc, DL, OutTy, Ops);
+}
+
bool
AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
unsigned RegWidth) {
return true;
}
+// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
+// of the string and obtains the integer values from them and combines these
+// into a single value to be used in the MRS/MSR instruction.
+static int getIntOperandFromRegisterString(StringRef RegString) {
+ SmallVector<StringRef, 5> Fields;
+ RegString.split(Fields, ":");
+
+ if (Fields.size() == 1)
+ return -1;
+
+ assert(Fields.size() == 5
+ && "Invalid number of fields in read register string");
+
+ SmallVector<int, 5> Ops;
+ bool AllIntFields = true;
+
+ for (StringRef Field : Fields) {
+ unsigned IntField;
+ AllIntFields &= !Field.getAsInteger(10, IntField);
+ Ops.push_back(IntField);
+ }
+
+ assert(AllIntFields &&
+ "Unexpected non-integer value in special register string.");
+
+ // Need to combine the integer fields of the string into a single value
+ // based on the bit encoding of MRS/MSR instruction.
+ return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
+ (Ops[3] << 3) | (Ops[4]);
+}
+
+// Lower the read_register intrinsic to an MRS instruction node if the special
+// register string argument is either of the form detailed in the ALCE (the
+// form described in getIntOperandsFromRegsterString) or is a named register
+// known by the MRS SysReg mapper.
+SDNode *AArch64DAGToDAGISel::SelectReadRegister(SDNode *N) {
+ const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
+ const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
+ SDLoc DL(N);
+
+ int Reg = getIntOperandFromRegisterString(RegString->getString());
+ if (Reg != -1)
+ return CurDAG->getMachineNode(AArch64::MRS, DL, N->getSimpleValueType(0),
+ MVT::Other,
+ CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+ N->getOperand(0));
+
+ // Use the sysreg mapper to map the remaining possible strings to the
+ // value for the register to be used for the instruction operand.
+ AArch64SysReg::MRSMapper mapper;
+ bool IsValidSpecialReg;
+ Reg = mapper.fromString(RegString->getString(),
+ Subtarget->getFeatureBits(),
+ IsValidSpecialReg);
+ if (IsValidSpecialReg)
+ return CurDAG->getMachineNode(AArch64::MRS, DL, N->getSimpleValueType(0),
+ MVT::Other,
+ CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+ N->getOperand(0));
+
+ return nullptr;
+}
+
+// Lower the write_register intrinsic to an MSR instruction node if the special
+// register string argument is either of the form detailed in the ALCE (the
+// form described in getIntOperandsFromRegsterString) or is a named register
+// known by the MSR SysReg mapper.
+SDNode *AArch64DAGToDAGISel::SelectWriteRegister(SDNode *N) {
+ const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
+ const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
+ SDLoc DL(N);
+
+ int Reg = getIntOperandFromRegisterString(RegString->getString());
+ if (Reg != -1)
+ return CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other,
+ CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+ N->getOperand(2), N->getOperand(0));
+
+ // Check if the register was one of those allowed as the pstatefield value in
+ // the MSR (immediate) instruction. To accept the values allowed in the
+ // pstatefield for the MSR (immediate) instruction, we also require that an
+ // immediate value has been provided as an argument, we know that this is
+ // the case as it has been ensured by semantic checking.
+ AArch64PState::PStateMapper PMapper;
+ bool IsValidSpecialReg;
+ Reg = PMapper.fromString(RegString->getString(),
+ Subtarget->getFeatureBits(),
+ IsValidSpecialReg);
+ if (IsValidSpecialReg) {
+ assert (isa<ConstantSDNode>(N->getOperand(2))
+ && "Expected a constant integer expression.");
+ uint64_t Immed = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
+ return CurDAG->getMachineNode(AArch64::MSRpstate, DL, MVT::Other,
+ CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+ CurDAG->getTargetConstant(Immed, DL, MVT::i16),
+ N->getOperand(0));
+ }
+
+ // Use the sysreg mapper to attempt to map the remaining possible strings
+ // to the value for the register to be used for the MSR (register)
+ // instruction operand.
+ AArch64SysReg::MSRMapper Mapper;
+ Reg = Mapper.fromString(RegString->getString(),
+ Subtarget->getFeatureBits(),
+ IsValidSpecialReg);
+
+ if (IsValidSpecialReg)
+ return CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other,
+ CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+ N->getOperand(2), N->getOperand(0));
+
+ return nullptr;
+}
+
SDNode *AArch64DAGToDAGISel::Select(SDNode *Node) {
// Dump information about the Node being selected
DEBUG(errs() << "Selecting: ");
default:
break;
+ case ISD::READ_REGISTER:
+ if (SDNode *Res = SelectReadRegister(Node))
+ return Res;
+ break;
+
+ case ISD::WRITE_REGISTER:
+ if (SDNode *Res = SelectWriteRegister(Node))
+ return Res;
+ break;
+
case ISD::ADD:
if (SDNode *I = SelectMLAV64LaneV128(Node))
return I;
int FI = cast<FrameIndexSDNode>(Node)->getIndex();
unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);
const TargetLowering *TLI = getTargetLowering();
- SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+ SDValue TFI = CurDAG->getTargetFrameIndex(
+ FI, TLI->getPointerTy(CurDAG->getDataLayout()));
SDLoc DL(Node);
SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32),
CurDAG->getTargetConstant(Shifter, DL, MVT::i32) };
break;
}
}
+ break;
}
case AArch64ISD::LD2post: {
if (VT == MVT::v8i8)
if (SDNode *I = SelectLIBM(Node))
return I;
break;
+
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT:
+ if (SDNode *I = SelectFPConvertWithRound(Node))
+ return I;
+ break;
}
// Select the default instruction
projects / oota-llvm.git / blobdiff