1 //===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines an instruction selector for the AArch64 target.
12 //===----------------------------------------------------------------------===//
14 #define DEBUG_TYPE "aarch64-isel"
16 #include "AArch64InstrInfo.h"
17 #include "AArch64Subtarget.h"
18 #include "AArch64TargetMachine.h"
19 #include "Utils/AArch64BaseInfo.h"
20 #include "llvm/ADT/APSInt.h"
21 #include "llvm/CodeGen/SelectionDAGISel.h"
22 #include "llvm/IR/GlobalValue.h"
23 #include "llvm/Support/Debug.h"
24 #include "llvm/Support/raw_ostream.h"
28 //===--------------------------------------------------------------------===//
29 /// AArch64 specific code to select AArch64 machine instructions for
30 /// SelectionDAG operations.
34 class AArch64DAGToDAGISel : public SelectionDAGISel {
35 AArch64TargetMachine &TM;
37 /// Keep a pointer to the AArch64Subtarget around so that we can
38 /// make the right decision when generating code for different targets.
39 const AArch64Subtarget *Subtarget;
42 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
43 CodeGenOpt::Level OptLevel)
44 : SelectionDAGISel(tm, OptLevel), TM(tm),
45 Subtarget(&TM.getSubtarget<AArch64Subtarget>()) {
48 virtual const char *getPassName() const {
49 return "AArch64 Instruction Selection";
52 // Include the pieces autogenerated from the target description.
53 #include "AArch64GenDAGISel.inc"
55 template<unsigned MemSize>
56 bool SelectOffsetUImm12(SDValue N, SDValue &UImm12) {
57 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
58 if (!CN || CN->getZExtValue() % MemSize != 0
59 || CN->getZExtValue() / MemSize > 0xfff)
62 UImm12 = CurDAG->getTargetConstant(CN->getZExtValue() / MemSize, MVT::i64);
66 template<unsigned RegWidth>
67 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
68 return SelectCVTFixedPosOperand(N, FixedPos, RegWidth);
71 /// Used for pre-lowered address-reference nodes, so we already know
72 /// the fields match. This operand's job is simply to add an
73 /// appropriate shift operand to the MOVZ/MOVK instruction.
74 template<unsigned LogShift>
75 bool SelectMOVWAddressRef(SDValue N, SDValue &Imm, SDValue &Shift) {
77 Shift = CurDAG->getTargetConstant(LogShift, MVT::i32);
81 bool SelectFPZeroOperand(SDValue N, SDValue &Dummy);
83 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
86 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
88 std::vector<SDValue> &OutOps);
90 bool SelectLogicalImm(SDValue N, SDValue &Imm);
92 template<unsigned RegWidth>
93 bool SelectTSTBOperand(SDValue N, SDValue &FixedPos) {
94 return SelectTSTBOperand(N, FixedPos, RegWidth);
97 bool SelectTSTBOperand(SDValue N, SDValue &FixedPos, unsigned RegWidth);
99 SDNode *SelectAtomic(SDNode *N, unsigned Op8, unsigned Op16, unsigned Op32,
102 /// Put the given constant into a pool and return a DAG which will give its
104 SDValue getConstantPoolItemAddress(SDLoc DL, const Constant *CV);
106 SDNode *TrySelectToMoveImm(SDNode *N);
107 SDNode *LowerToFPLitPool(SDNode *Node);
108 SDNode *SelectToLitPool(SDNode *N);
110 SDNode* Select(SDNode*);
116 AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
118 const ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N);
119 if (!CN) return false;
121 // An FCVT[SU] instruction performs: convertToInt(Val * 2^fbits) where fbits
122 // is between 1 and 32 for a destination w-register, or 1 and 64 for an
125 // By this stage, we've detected (fp_to_[su]int (fmul Val, THIS_NODE)) so we
126 // want THIS_NODE to be 2^fbits. This is much easier to deal with using
130 // fbits is between 1 and 64 in the worst-case, which means the fmul
131 // could have 2^64 as an actual operand. Need 65 bits of precision.
132 APSInt IntVal(65, true);
133 CN->getValueAPF().convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact);
135 // N.b. isPowerOf2 also checks for > 0.
136 if (!IsExact || !IntVal.isPowerOf2()) return false;
137 unsigned FBits = IntVal.logBase2();
139 // Checks above should have guaranteed that we haven't lost information in
140 // finding FBits, but it must still be in range.
141 if (FBits == 0 || FBits > RegWidth) return false;
143 FixedPos = CurDAG->getTargetConstant(64 - FBits, MVT::i32);
148 AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(const SDValue &Op,
150 std::vector<SDValue> &OutOps) {
151 switch (ConstraintCode) {
152 default: llvm_unreachable("Unrecognised AArch64 memory constraint");
154 // FIXME: more freedom is actually permitted for 'm'. We can go
155 // hunting for a base and an offset if we want. Of course, since
156 // we don't really know how the operand is going to be used we're
157 // probably restricted to the load/store pair's simm7 as an offset
160 OutOps.push_back(Op);
167 AArch64DAGToDAGISel::SelectFPZeroOperand(SDValue N, SDValue &Dummy) {
168 ConstantFPSDNode *Imm = dyn_cast<ConstantFPSDNode>(N);
169 if (!Imm || !Imm->getValueAPF().isPosZero())
172 // Doesn't actually carry any information, but keeps TableGen quiet.
173 Dummy = CurDAG->getTargetConstant(0, MVT::i32);
177 bool AArch64DAGToDAGISel::SelectLogicalImm(SDValue N, SDValue &Imm) {
179 uint32_t RegWidth = N.getValueType().getSizeInBits();
181 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
182 if (!CN) return false;
184 if (!A64Imms::isLogicalImm(RegWidth, CN->getZExtValue(), Bits))
187 Imm = CurDAG->getTargetConstant(Bits, MVT::i32);
191 SDNode *AArch64DAGToDAGISel::TrySelectToMoveImm(SDNode *Node) {
194 EVT DestType = Node->getValueType(0);
195 unsigned DestWidth = DestType.getSizeInBits();
200 uint32_t LogicalBits;
202 uint64_t BitPat = cast<ConstantSDNode>(Node)->getZExtValue();
203 if (A64Imms::isMOVZImm(DestWidth, BitPat, UImm16, Shift)) {
205 MOVOpcode = DestWidth == 64 ? AArch64::MOVZxii : AArch64::MOVZwii;
206 } else if (A64Imms::isMOVNImm(DestWidth, BitPat, UImm16, Shift)) {
208 MOVOpcode = DestWidth == 64 ? AArch64::MOVNxii : AArch64::MOVNwii;
209 } else if (DestWidth == 64 && A64Imms::isMOVNImm(32, BitPat, UImm16, Shift)) {
210 // To get something like 0x0000_0000_ffff_1234 into a 64-bit register we can
211 // use a 32-bit instruction: "movn w0, 0xedbc".
213 MOVOpcode = AArch64::MOVNwii;
214 } else if (A64Imms::isLogicalImm(DestWidth, BitPat, LogicalBits)) {
215 MOVOpcode = DestWidth == 64 ? AArch64::ORRxxi : AArch64::ORRwwi;
216 uint16_t ZR = DestWidth == 64 ? AArch64::XZR : AArch64::WZR;
218 return CurDAG->getMachineNode(MOVOpcode, dl, DestType,
219 CurDAG->getRegister(ZR, DestType),
220 CurDAG->getTargetConstant(LogicalBits, MVT::i32));
222 // Can't handle it in one instruction. There's scope for permitting two (or
223 // more) instructions, but that'll need more thought.
227 ResNode = CurDAG->getMachineNode(MOVOpcode, dl, MOVType,
228 CurDAG->getTargetConstant(UImm16, MVT::i32),
229 CurDAG->getTargetConstant(Shift, MVT::i32));
231 if (MOVType != DestType) {
232 ResNode = CurDAG->getMachineNode(TargetOpcode::SUBREG_TO_REG, dl,
233 MVT::i64, MVT::i32, MVT::Other,
234 CurDAG->getTargetConstant(0, MVT::i64),
236 CurDAG->getTargetConstant(AArch64::sub_32, MVT::i32));
243 AArch64DAGToDAGISel::getConstantPoolItemAddress(SDLoc DL,
244 const Constant *CV) {
245 EVT PtrVT = getTargetLowering()->getPointerTy();
247 switch (getTargetLowering()->getTargetMachine().getCodeModel()) {
248 case CodeModel::Small: {
250 getTargetLowering()->getDataLayout()->getABITypeAlignment(CV->getType());
251 return CurDAG->getNode(
252 AArch64ISD::WrapperSmall, DL, PtrVT,
253 CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_NO_FLAG),
254 CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_LO12),
255 CurDAG->getConstant(Alignment, MVT::i32));
257 case CodeModel::Large: {
259 LitAddr = CurDAG->getMachineNode(
260 AArch64::MOVZxii, DL, PtrVT,
261 CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G3),
262 CurDAG->getTargetConstant(3, MVT::i32));
263 LitAddr = CurDAG->getMachineNode(
264 AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0),
265 CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G2_NC),
266 CurDAG->getTargetConstant(2, MVT::i32));
267 LitAddr = CurDAG->getMachineNode(
268 AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0),
269 CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G1_NC),
270 CurDAG->getTargetConstant(1, MVT::i32));
271 LitAddr = CurDAG->getMachineNode(
272 AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0),
273 CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G0_NC),
274 CurDAG->getTargetConstant(0, MVT::i32));
275 return SDValue(LitAddr, 0);
278 llvm_unreachable("Only small and large code models supported now");
282 SDNode *AArch64DAGToDAGISel::SelectToLitPool(SDNode *Node) {
284 uint64_t UnsignedVal = cast<ConstantSDNode>(Node)->getZExtValue();
285 int64_t SignedVal = cast<ConstantSDNode>(Node)->getSExtValue();
286 EVT DestType = Node->getValueType(0);
288 // Since we may end up loading a 64-bit constant from a 32-bit entry the
289 // constant in the pool may have a different type to the eventual node.
290 ISD::LoadExtType Extension;
293 assert((DestType == MVT::i64 || DestType == MVT::i32)
294 && "Only expect integer constants at the moment");
296 if (DestType == MVT::i32) {
297 Extension = ISD::NON_EXTLOAD;
299 } else if (UnsignedVal <= UINT32_MAX) {
300 Extension = ISD::ZEXTLOAD;
302 } else if (SignedVal >= INT32_MIN && SignedVal <= INT32_MAX) {
303 Extension = ISD::SEXTLOAD;
306 Extension = ISD::NON_EXTLOAD;
310 Constant *CV = ConstantInt::get(Type::getIntNTy(*CurDAG->getContext(),
311 MemType.getSizeInBits()),
313 SDValue PoolAddr = getConstantPoolItemAddress(DL, CV);
315 getTargetLowering()->getDataLayout()->getABITypeAlignment(CV->getType());
317 return CurDAG->getExtLoad(Extension, DL, DestType, CurDAG->getEntryNode(),
319 MachinePointerInfo::getConstantPool(), MemType,
320 /* isVolatile = */ false,
321 /* isNonTemporal = */ false,
322 Alignment).getNode();
325 SDNode *AArch64DAGToDAGISel::LowerToFPLitPool(SDNode *Node) {
327 const ConstantFP *FV = cast<ConstantFPSDNode>(Node)->getConstantFPValue();
328 EVT DestType = Node->getValueType(0);
331 getTargetLowering()->getDataLayout()->getABITypeAlignment(FV->getType());
332 SDValue PoolAddr = getConstantPoolItemAddress(DL, FV);
334 return CurDAG->getLoad(DestType, DL, CurDAG->getEntryNode(), PoolAddr,
335 MachinePointerInfo::getConstantPool(),
336 /* isVolatile = */ false,
337 /* isNonTemporal = */ false,
338 /* isInvariant = */ true,
339 Alignment).getNode();
343 AArch64DAGToDAGISel::SelectTSTBOperand(SDValue N, SDValue &FixedPos,
345 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
346 if (!CN) return false;
348 uint64_t Val = CN->getZExtValue();
350 if (!isPowerOf2_64(Val)) return false;
352 unsigned TestedBit = Log2_64(Val);
353 // Checks above should have guaranteed that we haven't lost information in
354 // finding TestedBit, but it must still be in range.
355 if (TestedBit >= RegWidth) return false;
357 FixedPos = CurDAG->getTargetConstant(TestedBit, MVT::i64);
361 SDNode *AArch64DAGToDAGISel::SelectAtomic(SDNode *Node, unsigned Op8,
362 unsigned Op16,unsigned Op32,
364 // Mostly direct translation to the given operations, except that we preserve
365 // the AtomicOrdering for use later on.
366 AtomicSDNode *AN = cast<AtomicSDNode>(Node);
367 EVT VT = AN->getMemoryVT();
372 else if (VT == MVT::i16)
374 else if (VT == MVT::i32)
376 else if (VT == MVT::i64)
379 llvm_unreachable("Unexpected atomic operation");
381 SmallVector<SDValue, 4> Ops;
382 for (unsigned i = 1; i < AN->getNumOperands(); ++i)
383 Ops.push_back(AN->getOperand(i));
385 Ops.push_back(CurDAG->getTargetConstant(AN->getOrdering(), MVT::i32));
386 Ops.push_back(AN->getOperand(0)); // Chain moves to the end
388 return CurDAG->SelectNodeTo(Node, Op,
389 AN->getValueType(0), MVT::Other,
390 &Ops[0], Ops.size());
393 SDNode *AArch64DAGToDAGISel::Select(SDNode *Node) {
394 // Dump information about the Node being selected
395 DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << "\n");
397 if (Node->isMachineOpcode()) {
398 DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
403 switch (Node->getOpcode()) {
404 case ISD::ATOMIC_LOAD_ADD:
405 return SelectAtomic(Node,
406 AArch64::ATOMIC_LOAD_ADD_I8,
407 AArch64::ATOMIC_LOAD_ADD_I16,
408 AArch64::ATOMIC_LOAD_ADD_I32,
409 AArch64::ATOMIC_LOAD_ADD_I64);
410 case ISD::ATOMIC_LOAD_SUB:
411 return SelectAtomic(Node,
412 AArch64::ATOMIC_LOAD_SUB_I8,
413 AArch64::ATOMIC_LOAD_SUB_I16,
414 AArch64::ATOMIC_LOAD_SUB_I32,
415 AArch64::ATOMIC_LOAD_SUB_I64);
416 case ISD::ATOMIC_LOAD_AND:
417 return SelectAtomic(Node,
418 AArch64::ATOMIC_LOAD_AND_I8,
419 AArch64::ATOMIC_LOAD_AND_I16,
420 AArch64::ATOMIC_LOAD_AND_I32,
421 AArch64::ATOMIC_LOAD_AND_I64);
422 case ISD::ATOMIC_LOAD_OR:
423 return SelectAtomic(Node,
424 AArch64::ATOMIC_LOAD_OR_I8,
425 AArch64::ATOMIC_LOAD_OR_I16,
426 AArch64::ATOMIC_LOAD_OR_I32,
427 AArch64::ATOMIC_LOAD_OR_I64);
428 case ISD::ATOMIC_LOAD_XOR:
429 return SelectAtomic(Node,
430 AArch64::ATOMIC_LOAD_XOR_I8,
431 AArch64::ATOMIC_LOAD_XOR_I16,
432 AArch64::ATOMIC_LOAD_XOR_I32,
433 AArch64::ATOMIC_LOAD_XOR_I64);
434 case ISD::ATOMIC_LOAD_NAND:
435 return SelectAtomic(Node,
436 AArch64::ATOMIC_LOAD_NAND_I8,
437 AArch64::ATOMIC_LOAD_NAND_I16,
438 AArch64::ATOMIC_LOAD_NAND_I32,
439 AArch64::ATOMIC_LOAD_NAND_I64);
440 case ISD::ATOMIC_LOAD_MIN:
441 return SelectAtomic(Node,
442 AArch64::ATOMIC_LOAD_MIN_I8,
443 AArch64::ATOMIC_LOAD_MIN_I16,
444 AArch64::ATOMIC_LOAD_MIN_I32,
445 AArch64::ATOMIC_LOAD_MIN_I64);
446 case ISD::ATOMIC_LOAD_MAX:
447 return SelectAtomic(Node,
448 AArch64::ATOMIC_LOAD_MAX_I8,
449 AArch64::ATOMIC_LOAD_MAX_I16,
450 AArch64::ATOMIC_LOAD_MAX_I32,
451 AArch64::ATOMIC_LOAD_MAX_I64);
452 case ISD::ATOMIC_LOAD_UMIN:
453 return SelectAtomic(Node,
454 AArch64::ATOMIC_LOAD_UMIN_I8,
455 AArch64::ATOMIC_LOAD_UMIN_I16,
456 AArch64::ATOMIC_LOAD_UMIN_I32,
457 AArch64::ATOMIC_LOAD_UMIN_I64);
458 case ISD::ATOMIC_LOAD_UMAX:
459 return SelectAtomic(Node,
460 AArch64::ATOMIC_LOAD_UMAX_I8,
461 AArch64::ATOMIC_LOAD_UMAX_I16,
462 AArch64::ATOMIC_LOAD_UMAX_I32,
463 AArch64::ATOMIC_LOAD_UMAX_I64);
464 case ISD::ATOMIC_SWAP:
465 return SelectAtomic(Node,
466 AArch64::ATOMIC_SWAP_I8,
467 AArch64::ATOMIC_SWAP_I16,
468 AArch64::ATOMIC_SWAP_I32,
469 AArch64::ATOMIC_SWAP_I64);
470 case ISD::ATOMIC_CMP_SWAP:
471 return SelectAtomic(Node,
472 AArch64::ATOMIC_CMP_SWAP_I8,
473 AArch64::ATOMIC_CMP_SWAP_I16,
474 AArch64::ATOMIC_CMP_SWAP_I32,
475 AArch64::ATOMIC_CMP_SWAP_I64);
476 case ISD::FrameIndex: {
477 int FI = cast<FrameIndexSDNode>(Node)->getIndex();
478 EVT PtrTy = getTargetLowering()->getPointerTy();
479 SDValue TFI = CurDAG->getTargetFrameIndex(FI, PtrTy);
480 return CurDAG->SelectNodeTo(Node, AArch64::ADDxxi_lsl0_s, PtrTy,
481 TFI, CurDAG->getTargetConstant(0, PtrTy));
483 case ISD::ConstantPool: {
484 // Constant pools are fine, just create a Target entry.
485 ConstantPoolSDNode *CN = cast<ConstantPoolSDNode>(Node);
486 const Constant *C = CN->getConstVal();
487 SDValue CP = CurDAG->getTargetConstantPool(C, CN->getValueType(0));
489 ReplaceUses(SDValue(Node, 0), CP);
492 case ISD::Constant: {
494 if (cast<ConstantSDNode>(Node)->getZExtValue() == 0) {
495 // XZR and WZR are probably even better than an actual move: most of the
496 // time they can be folded into another instruction with *no* cost.
498 EVT Ty = Node->getValueType(0);
499 assert((Ty == MVT::i32 || Ty == MVT::i64) && "unexpected type");
500 uint16_t Register = Ty == MVT::i32 ? AArch64::WZR : AArch64::XZR;
501 ResNode = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
503 Register, Ty).getNode();
506 // Next best option is a move-immediate, see if we can do that.
508 ResNode = TrySelectToMoveImm(Node);
514 // If even that fails we fall back to a lit-pool entry at the moment. Future
515 // tuning may change this to a sequence of MOVZ/MOVN/MOVK instructions.
516 ResNode = SelectToLitPool(Node);
517 assert(ResNode && "We need *some* way to materialise a constant");
519 // We want to continue selection at this point since the litpool access
520 // generated used generic nodes for simplicity.
521 ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0));
525 case ISD::ConstantFP: {
526 if (A64Imms::isFPImm(cast<ConstantFPSDNode>(Node)->getValueAPF())) {
527 // FMOV will take care of it from TableGen
531 SDNode *ResNode = LowerToFPLitPool(Node);
532 ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0));
534 // We want to continue selection at this point since the litpool access
535 // generated used generic nodes for simplicity.
540 break; // Let generic code handle it
543 SDNode *ResNode = SelectCode(Node);
545 DEBUG(dbgs() << "=> ";
546 if (ResNode == NULL || ResNode == Node)
549 ResNode->dump(CurDAG);
555 /// This pass converts a legalized DAG into a AArch64-specific DAG, ready for
556 /// instruction scheduling.
557 FunctionPass *llvm::createAArch64ISelDAG(AArch64TargetMachine &TM,
558 CodeGenOpt::Level OptLevel) {
559 return new AArch64DAGToDAGISel(TM, OptLevel);