1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the AArch64 implementation of the TargetInstrInfo class.
12 //===----------------------------------------------------------------------===//
14 #include "AArch64InstrInfo.h"
15 #include "AArch64MachineCombinerPattern.h"
16 #include "AArch64Subtarget.h"
17 #include "MCTargetDesc/AArch64AddressingModes.h"
18 #include "llvm/CodeGen/MachineFrameInfo.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/MachineMemOperand.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/PseudoSourceValue.h"
23 #include "llvm/MC/MCInst.h"
24 #include "llvm/Support/ErrorHandling.h"
25 #include "llvm/Support/TargetRegistry.h"
29 #define GET_INSTRINFO_CTOR_DTOR
30 #include "AArch64GenInstrInfo.inc"
32 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
33 : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP),
34 RI(STI.getTargetTriple()), Subtarget(STI) {}
36 /// GetInstSize - Return the number of bytes of code the specified
37 /// instruction may be. This returns the maximum number of bytes.
38 unsigned AArch64InstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const {
39 const MachineBasicBlock &MBB = *MI->getParent();
40 const MachineFunction *MF = MBB.getParent();
41 const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
43 if (MI->getOpcode() == AArch64::INLINEASM)
44 return getInlineAsmLength(MI->getOperand(0).getSymbolName(), *MAI);
46 const MCInstrDesc &Desc = MI->getDesc();
47 switch (Desc.getOpcode()) {
49 // Anything not explicitly designated otherwise is a nomal 4-byte insn.
51 case TargetOpcode::DBG_VALUE:
52 case TargetOpcode::EH_LABEL:
53 case TargetOpcode::IMPLICIT_DEF:
54 case TargetOpcode::KILL:
58 llvm_unreachable("GetInstSizeInBytes()- Unable to determin insn size");
61 static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
62 SmallVectorImpl<MachineOperand> &Cond) {
63 // Block ends with fall-through condbranch.
64 switch (LastInst->getOpcode()) {
66 llvm_unreachable("Unknown branch instruction?");
68 Target = LastInst->getOperand(1).getMBB();
69 Cond.push_back(LastInst->getOperand(0));
75 Target = LastInst->getOperand(1).getMBB();
76 Cond.push_back(MachineOperand::CreateImm(-1));
77 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
78 Cond.push_back(LastInst->getOperand(0));
84 Target = LastInst->getOperand(2).getMBB();
85 Cond.push_back(MachineOperand::CreateImm(-1));
86 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
87 Cond.push_back(LastInst->getOperand(0));
88 Cond.push_back(LastInst->getOperand(1));
93 bool AArch64InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
94 MachineBasicBlock *&TBB,
95 MachineBasicBlock *&FBB,
96 SmallVectorImpl<MachineOperand> &Cond,
97 bool AllowModify) const {
98 // If the block has no terminators, it just falls into the block after it.
99 MachineBasicBlock::iterator I = MBB.end();
100 if (I == MBB.begin())
103 while (I->isDebugValue()) {
104 if (I == MBB.begin())
108 if (!isUnpredicatedTerminator(I))
111 // Get the last instruction in the block.
112 MachineInstr *LastInst = I;
114 // If there is only one terminator instruction, process it.
115 unsigned LastOpc = LastInst->getOpcode();
116 if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
117 if (isUncondBranchOpcode(LastOpc)) {
118 TBB = LastInst->getOperand(0).getMBB();
121 if (isCondBranchOpcode(LastOpc)) {
122 // Block ends with fall-through condbranch.
123 parseCondBranch(LastInst, TBB, Cond);
126 return true; // Can't handle indirect branch.
129 // Get the instruction before it if it is a terminator.
130 MachineInstr *SecondLastInst = I;
131 unsigned SecondLastOpc = SecondLastInst->getOpcode();
133 // If AllowModify is true and the block ends with two or more unconditional
134 // branches, delete all but the first unconditional branch.
135 if (AllowModify && isUncondBranchOpcode(LastOpc)) {
136 while (isUncondBranchOpcode(SecondLastOpc)) {
137 LastInst->eraseFromParent();
138 LastInst = SecondLastInst;
139 LastOpc = LastInst->getOpcode();
140 if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
141 // Return now the only terminator is an unconditional branch.
142 TBB = LastInst->getOperand(0).getMBB();
146 SecondLastOpc = SecondLastInst->getOpcode();
151 // If there are three terminators, we don't know what sort of block this is.
152 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(--I))
155 // If the block ends with a B and a Bcc, handle it.
156 if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
157 parseCondBranch(SecondLastInst, TBB, Cond);
158 FBB = LastInst->getOperand(0).getMBB();
162 // If the block ends with two unconditional branches, handle it. The second
163 // one is not executed, so remove it.
164 if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
165 TBB = SecondLastInst->getOperand(0).getMBB();
168 I->eraseFromParent();
172 // ...likewise if it ends with an indirect branch followed by an unconditional
174 if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
177 I->eraseFromParent();
181 // Otherwise, can't handle this.
185 bool AArch64InstrInfo::ReverseBranchCondition(
186 SmallVectorImpl<MachineOperand> &Cond) const {
187 if (Cond[0].getImm() != -1) {
189 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
190 Cond[0].setImm(AArch64CC::getInvertedCondCode(CC));
192 // Folded compare-and-branch
193 switch (Cond[1].getImm()) {
195 llvm_unreachable("Unknown conditional branch!");
197 Cond[1].setImm(AArch64::CBNZW);
200 Cond[1].setImm(AArch64::CBZW);
203 Cond[1].setImm(AArch64::CBNZX);
206 Cond[1].setImm(AArch64::CBZX);
209 Cond[1].setImm(AArch64::TBNZW);
212 Cond[1].setImm(AArch64::TBZW);
215 Cond[1].setImm(AArch64::TBNZX);
218 Cond[1].setImm(AArch64::TBZX);
226 unsigned AArch64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
227 MachineBasicBlock::iterator I = MBB.end();
228 if (I == MBB.begin())
231 while (I->isDebugValue()) {
232 if (I == MBB.begin())
236 if (!isUncondBranchOpcode(I->getOpcode()) &&
237 !isCondBranchOpcode(I->getOpcode()))
240 // Remove the branch.
241 I->eraseFromParent();
245 if (I == MBB.begin())
248 if (!isCondBranchOpcode(I->getOpcode()))
251 // Remove the branch.
252 I->eraseFromParent();
256 void AArch64InstrInfo::instantiateCondBranch(
257 MachineBasicBlock &MBB, DebugLoc DL, MachineBasicBlock *TBB,
258 const SmallVectorImpl<MachineOperand> &Cond) const {
259 if (Cond[0].getImm() != -1) {
261 BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
263 // Folded compare-and-branch
264 // Note that we use addOperand instead of addReg to keep the flags.
265 const MachineInstrBuilder MIB =
266 BuildMI(&MBB, DL, get(Cond[1].getImm())).addOperand(Cond[2]);
268 MIB.addImm(Cond[3].getImm());
273 unsigned AArch64InstrInfo::InsertBranch(
274 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
275 const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const {
276 // Shouldn't be a fall through.
277 assert(TBB && "InsertBranch must not be told to insert a fallthrough");
280 if (Cond.empty()) // Unconditional branch?
281 BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
283 instantiateCondBranch(MBB, DL, TBB, Cond);
287 // Two-way conditional branch.
288 instantiateCondBranch(MBB, DL, TBB, Cond);
289 BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
293 // Find the original register that VReg is copied from.
294 static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
295 while (TargetRegisterInfo::isVirtualRegister(VReg)) {
296 const MachineInstr *DefMI = MRI.getVRegDef(VReg);
297 if (!DefMI->isFullCopy())
299 VReg = DefMI->getOperand(1).getReg();
304 // Determine if VReg is defined by an instruction that can be folded into a
305 // csel instruction. If so, return the folded opcode, and the replacement
307 static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
308 unsigned *NewVReg = nullptr) {
309 VReg = removeCopies(MRI, VReg);
310 if (!TargetRegisterInfo::isVirtualRegister(VReg))
313 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
314 const MachineInstr *DefMI = MRI.getVRegDef(VReg);
316 unsigned SrcOpNum = 0;
317 switch (DefMI->getOpcode()) {
318 case AArch64::ADDSXri:
319 case AArch64::ADDSWri:
320 // if NZCV is used, do not fold.
321 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
323 // fall-through to ADDXri and ADDWri.
324 case AArch64::ADDXri:
325 case AArch64::ADDWri:
326 // add x, 1 -> csinc.
327 if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
328 DefMI->getOperand(3).getImm() != 0)
331 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
334 case AArch64::ORNXrr:
335 case AArch64::ORNWrr: {
336 // not x -> csinv, represented as orn dst, xzr, src.
337 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
338 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
341 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
345 case AArch64::SUBSXrr:
346 case AArch64::SUBSWrr:
347 // if NZCV is used, do not fold.
348 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
350 // fall-through to SUBXrr and SUBWrr.
351 case AArch64::SUBXrr:
352 case AArch64::SUBWrr: {
353 // neg x -> csneg, represented as sub dst, xzr, src.
354 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
355 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
358 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
364 assert(Opc && SrcOpNum && "Missing parameters");
367 *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
371 bool AArch64InstrInfo::canInsertSelect(
372 const MachineBasicBlock &MBB, const SmallVectorImpl<MachineOperand> &Cond,
373 unsigned TrueReg, unsigned FalseReg, int &CondCycles, int &TrueCycles,
374 int &FalseCycles) const {
375 // Check register classes.
376 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
377 const TargetRegisterClass *RC =
378 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
382 // Expanding cbz/tbz requires an extra cycle of latency on the condition.
383 unsigned ExtraCondLat = Cond.size() != 1;
385 // GPRs are handled by csel.
386 // FIXME: Fold in x+1, -x, and ~x when applicable.
387 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
388 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
389 // Single-cycle csel, csinc, csinv, and csneg.
390 CondCycles = 1 + ExtraCondLat;
391 TrueCycles = FalseCycles = 1;
392 if (canFoldIntoCSel(MRI, TrueReg))
394 else if (canFoldIntoCSel(MRI, FalseReg))
399 // Scalar floating point is handled by fcsel.
400 // FIXME: Form fabs, fmin, and fmax when applicable.
401 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
402 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
403 CondCycles = 5 + ExtraCondLat;
404 TrueCycles = FalseCycles = 2;
412 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
413 MachineBasicBlock::iterator I, DebugLoc DL,
415 const SmallVectorImpl<MachineOperand> &Cond,
416 unsigned TrueReg, unsigned FalseReg) const {
417 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
419 // Parse the condition code, see parseCondBranch() above.
420 AArch64CC::CondCode CC;
421 switch (Cond.size()) {
423 llvm_unreachable("Unknown condition opcode in Cond");
425 CC = AArch64CC::CondCode(Cond[0].getImm());
427 case 3: { // cbz/cbnz
428 // We must insert a compare against 0.
430 switch (Cond[1].getImm()) {
432 llvm_unreachable("Unknown branch opcode in Cond");
450 unsigned SrcReg = Cond[2].getReg();
452 // cmp reg, #0 is actually subs xzr, reg, #0.
453 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
454 BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
459 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
460 BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
467 case 4: { // tbz/tbnz
468 // We must insert a tst instruction.
469 switch (Cond[1].getImm()) {
471 llvm_unreachable("Unknown branch opcode in Cond");
481 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
482 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
483 BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
484 .addReg(Cond[2].getReg())
486 AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
488 BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
489 .addReg(Cond[2].getReg())
491 AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
497 const TargetRegisterClass *RC = nullptr;
498 bool TryFold = false;
499 if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
500 RC = &AArch64::GPR64RegClass;
501 Opc = AArch64::CSELXr;
503 } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
504 RC = &AArch64::GPR32RegClass;
505 Opc = AArch64::CSELWr;
507 } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
508 RC = &AArch64::FPR64RegClass;
509 Opc = AArch64::FCSELDrrr;
510 } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
511 RC = &AArch64::FPR32RegClass;
512 Opc = AArch64::FCSELSrrr;
514 assert(RC && "Unsupported regclass");
516 // Try folding simple instructions into the csel.
518 unsigned NewVReg = 0;
519 unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
521 // The folded opcodes csinc, csinc and csneg apply the operation to
522 // FalseReg, so we need to invert the condition.
523 CC = AArch64CC::getInvertedCondCode(CC);
526 FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
528 // Fold the operation. Leave any dead instructions for DCE to clean up.
532 // The extends the live range of NewVReg.
533 MRI.clearKillFlags(NewVReg);
537 // Pull all virtual register into the appropriate class.
538 MRI.constrainRegClass(TrueReg, RC);
539 MRI.constrainRegClass(FalseReg, RC);
542 BuildMI(MBB, I, DL, get(Opc), DstReg).addReg(TrueReg).addReg(FalseReg).addImm(
546 // FIXME: this implementation should be micro-architecture dependent, so a
547 // micro-architecture target hook should be introduced here in future.
548 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr *MI) const {
549 if (!Subtarget.isCortexA57() && !Subtarget.isCortexA53())
550 return MI->isAsCheapAsAMove();
552 switch (MI->getOpcode()) {
556 // add/sub on register without shift
557 case AArch64::ADDWri:
558 case AArch64::ADDXri:
559 case AArch64::SUBWri:
560 case AArch64::SUBXri:
561 return (MI->getOperand(3).getImm() == 0);
563 // logical ops on immediate
564 case AArch64::ANDWri:
565 case AArch64::ANDXri:
566 case AArch64::EORWri:
567 case AArch64::EORXri:
568 case AArch64::ORRWri:
569 case AArch64::ORRXri:
572 // logical ops on register without shift
573 case AArch64::ANDWrr:
574 case AArch64::ANDXrr:
575 case AArch64::BICWrr:
576 case AArch64::BICXrr:
577 case AArch64::EONWrr:
578 case AArch64::EONXrr:
579 case AArch64::EORWrr:
580 case AArch64::EORXrr:
581 case AArch64::ORNWrr:
582 case AArch64::ORNXrr:
583 case AArch64::ORRWrr:
584 case AArch64::ORRXrr:
588 llvm_unreachable("Unknown opcode to check as cheap as a move!");
591 bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
592 unsigned &SrcReg, unsigned &DstReg,
593 unsigned &SubIdx) const {
594 switch (MI.getOpcode()) {
597 case AArch64::SBFMXri: // aka sxtw
598 case AArch64::UBFMXri: // aka uxtw
599 // Check for the 32 -> 64 bit extension case, these instructions can do
601 if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
603 // This is a signed or unsigned 32 -> 64 bit extension.
604 SrcReg = MI.getOperand(1).getReg();
605 DstReg = MI.getOperand(0).getReg();
606 SubIdx = AArch64::sub_32;
612 AArch64InstrInfo::areMemAccessesTriviallyDisjoint(MachineInstr *MIa,
614 AliasAnalysis *AA) const {
615 const TargetRegisterInfo *TRI = &getRegisterInfo();
616 unsigned BaseRegA = 0, BaseRegB = 0;
617 int OffsetA = 0, OffsetB = 0;
618 int WidthA = 0, WidthB = 0;
620 assert(MIa && MIa->mayLoadOrStore() && "MIa must be a load or store.");
621 assert(MIb && MIb->mayLoadOrStore() && "MIb must be a load or store.");
623 if (MIa->hasUnmodeledSideEffects() || MIb->hasUnmodeledSideEffects() ||
624 MIa->hasOrderedMemoryRef() || MIb->hasOrderedMemoryRef())
627 // Retrieve the base register, offset from the base register and width. Width
628 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
629 // base registers are identical, and the offset of a lower memory access +
630 // the width doesn't overlap the offset of a higher memory access,
631 // then the memory accesses are different.
632 if (getLdStBaseRegImmOfsWidth(MIa, BaseRegA, OffsetA, WidthA, TRI) &&
633 getLdStBaseRegImmOfsWidth(MIb, BaseRegB, OffsetB, WidthB, TRI)) {
634 if (BaseRegA == BaseRegB) {
635 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
636 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
637 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
638 if (LowOffset + LowWidth <= HighOffset)
645 /// analyzeCompare - For a comparison instruction, return the source registers
646 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
647 /// Return true if the comparison instruction can be analyzed.
648 bool AArch64InstrInfo::analyzeCompare(const MachineInstr *MI, unsigned &SrcReg,
649 unsigned &SrcReg2, int &CmpMask,
650 int &CmpValue) const {
651 switch (MI->getOpcode()) {
654 case AArch64::SUBSWrr:
655 case AArch64::SUBSWrs:
656 case AArch64::SUBSWrx:
657 case AArch64::SUBSXrr:
658 case AArch64::SUBSXrs:
659 case AArch64::SUBSXrx:
660 case AArch64::ADDSWrr:
661 case AArch64::ADDSWrs:
662 case AArch64::ADDSWrx:
663 case AArch64::ADDSXrr:
664 case AArch64::ADDSXrs:
665 case AArch64::ADDSXrx:
666 // Replace SUBSWrr with SUBWrr if NZCV is not used.
667 SrcReg = MI->getOperand(1).getReg();
668 SrcReg2 = MI->getOperand(2).getReg();
672 case AArch64::SUBSWri:
673 case AArch64::ADDSWri:
674 case AArch64::SUBSXri:
675 case AArch64::ADDSXri:
676 SrcReg = MI->getOperand(1).getReg();
679 // FIXME: In order to convert CmpValue to 0 or 1
680 CmpValue = (MI->getOperand(2).getImm() != 0);
682 case AArch64::ANDSWri:
683 case AArch64::ANDSXri:
684 // ANDS does not use the same encoding scheme as the others xxxS
686 SrcReg = MI->getOperand(1).getReg();
689 // FIXME:The return val type of decodeLogicalImmediate is uint64_t,
690 // while the type of CmpValue is int. When converting uint64_t to int,
691 // the high 32 bits of uint64_t will be lost.
692 // In fact it causes a bug in spec2006-483.xalancbmk
693 // CmpValue is only used to compare with zero in OptimizeCompareInstr
694 CmpValue = (AArch64_AM::decodeLogicalImmediate(
695 MI->getOperand(2).getImm(),
696 MI->getOpcode() == AArch64::ANDSWri ? 32 : 64) != 0);
703 static bool UpdateOperandRegClass(MachineInstr *Instr) {
704 MachineBasicBlock *MBB = Instr->getParent();
705 assert(MBB && "Can't get MachineBasicBlock here");
706 MachineFunction *MF = MBB->getParent();
707 assert(MF && "Can't get MachineFunction here");
708 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
709 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
710 MachineRegisterInfo *MRI = &MF->getRegInfo();
712 for (unsigned OpIdx = 0, EndIdx = Instr->getNumOperands(); OpIdx < EndIdx;
714 MachineOperand &MO = Instr->getOperand(OpIdx);
715 const TargetRegisterClass *OpRegCstraints =
716 Instr->getRegClassConstraint(OpIdx, TII, TRI);
718 // If there's no constraint, there's nothing to do.
721 // If the operand is a frame index, there's nothing to do here.
722 // A frame index operand will resolve correctly during PEI.
727 "Operand has register constraints without being a register!");
729 unsigned Reg = MO.getReg();
730 if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
731 if (!OpRegCstraints->contains(Reg))
733 } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
734 !MRI->constrainRegClass(Reg, OpRegCstraints))
741 /// \brief Return the opcode that does not set flags when possible - otherwise
742 /// return the original opcode. The caller is responsible to do the actual
743 /// substitution and legality checking.
744 static unsigned convertFlagSettingOpcode(const MachineInstr *MI) {
745 // Don't convert all compare instructions, because for some the zero register
746 // encoding becomes the sp register.
747 bool MIDefinesZeroReg = false;
748 if (MI->definesRegister(AArch64::WZR) || MI->definesRegister(AArch64::XZR))
749 MIDefinesZeroReg = true;
751 switch (MI->getOpcode()) {
753 return MI->getOpcode();
754 case AArch64::ADDSWrr:
755 return AArch64::ADDWrr;
756 case AArch64::ADDSWri:
757 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
758 case AArch64::ADDSWrs:
759 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
760 case AArch64::ADDSWrx:
761 return AArch64::ADDWrx;
762 case AArch64::ADDSXrr:
763 return AArch64::ADDXrr;
764 case AArch64::ADDSXri:
765 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
766 case AArch64::ADDSXrs:
767 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
768 case AArch64::ADDSXrx:
769 return AArch64::ADDXrx;
770 case AArch64::SUBSWrr:
771 return AArch64::SUBWrr;
772 case AArch64::SUBSWri:
773 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
774 case AArch64::SUBSWrs:
775 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
776 case AArch64::SUBSWrx:
777 return AArch64::SUBWrx;
778 case AArch64::SUBSXrr:
779 return AArch64::SUBXrr;
780 case AArch64::SUBSXri:
781 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
782 case AArch64::SUBSXrs:
783 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
784 case AArch64::SUBSXrx:
785 return AArch64::SUBXrx;
789 /// True when condition code could be modified on the instruction
790 /// trace starting at from and ending at to.
791 static bool modifiesConditionCode(MachineInstr *From, MachineInstr *To,
792 const bool CheckOnlyCCWrites,
793 const TargetRegisterInfo *TRI) {
794 // We iterate backward starting \p To until we hit \p From
795 MachineBasicBlock::iterator I = To, E = From, B = To->getParent()->begin();
797 // Early exit if To is at the beginning of the BB.
801 // Check whether the definition of SrcReg is in the same basic block as
802 // Compare. If not, assume the condition code gets modified on some path.
803 if (To->getParent() != From->getParent())
806 // Check that NZCV isn't set on the trace.
807 for (--I; I != E; --I) {
808 const MachineInstr &Instr = *I;
810 if (Instr.modifiesRegister(AArch64::NZCV, TRI) ||
811 (!CheckOnlyCCWrites && Instr.readsRegister(AArch64::NZCV, TRI)))
812 // This instruction modifies or uses NZCV after the one we want to
816 // We currently don't allow the instruction trace to cross basic
822 /// optimizeCompareInstr - Convert the instruction supplying the argument to the
823 /// comparison into one that sets the zero bit in the flags register.
824 bool AArch64InstrInfo::optimizeCompareInstr(
825 MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
826 int CmpValue, const MachineRegisterInfo *MRI) const {
828 // Replace SUBSWrr with SUBWrr if NZCV is not used.
829 int Cmp_NZCV = CmpInstr->findRegisterDefOperandIdx(AArch64::NZCV, true);
830 if (Cmp_NZCV != -1) {
831 if (CmpInstr->definesRegister(AArch64::WZR) ||
832 CmpInstr->definesRegister(AArch64::XZR)) {
833 CmpInstr->eraseFromParent();
836 unsigned Opc = CmpInstr->getOpcode();
837 unsigned NewOpc = convertFlagSettingOpcode(CmpInstr);
840 const MCInstrDesc &MCID = get(NewOpc);
841 CmpInstr->setDesc(MCID);
842 CmpInstr->RemoveOperand(Cmp_NZCV);
843 bool succeeded = UpdateOperandRegClass(CmpInstr);
845 assert(succeeded && "Some operands reg class are incompatible!");
849 // Continue only if we have a "ri" where immediate is zero.
850 // FIXME:CmpValue has already been converted to 0 or 1 in analyzeCompare
852 assert((CmpValue == 0 || CmpValue == 1) && "CmpValue must be 0 or 1!");
853 if (CmpValue != 0 || SrcReg2 != 0)
856 // CmpInstr is a Compare instruction if destination register is not used.
857 if (!MRI->use_nodbg_empty(CmpInstr->getOperand(0).getReg()))
860 // Get the unique definition of SrcReg.
861 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
865 bool CheckOnlyCCWrites = false;
866 const TargetRegisterInfo *TRI = &getRegisterInfo();
867 if (modifiesConditionCode(MI, CmpInstr, CheckOnlyCCWrites, TRI))
870 unsigned NewOpc = MI->getOpcode();
871 switch (MI->getOpcode()) {
874 case AArch64::ADDSWrr:
875 case AArch64::ADDSWri:
876 case AArch64::ADDSXrr:
877 case AArch64::ADDSXri:
878 case AArch64::SUBSWrr:
879 case AArch64::SUBSWri:
880 case AArch64::SUBSXrr:
881 case AArch64::SUBSXri:
883 case AArch64::ADDWrr: NewOpc = AArch64::ADDSWrr; break;
884 case AArch64::ADDWri: NewOpc = AArch64::ADDSWri; break;
885 case AArch64::ADDXrr: NewOpc = AArch64::ADDSXrr; break;
886 case AArch64::ADDXri: NewOpc = AArch64::ADDSXri; break;
887 case AArch64::ADCWr: NewOpc = AArch64::ADCSWr; break;
888 case AArch64::ADCXr: NewOpc = AArch64::ADCSXr; break;
889 case AArch64::SUBWrr: NewOpc = AArch64::SUBSWrr; break;
890 case AArch64::SUBWri: NewOpc = AArch64::SUBSWri; break;
891 case AArch64::SUBXrr: NewOpc = AArch64::SUBSXrr; break;
892 case AArch64::SUBXri: NewOpc = AArch64::SUBSXri; break;
893 case AArch64::SBCWr: NewOpc = AArch64::SBCSWr; break;
894 case AArch64::SBCXr: NewOpc = AArch64::SBCSXr; break;
895 case AArch64::ANDWri: NewOpc = AArch64::ANDSWri; break;
896 case AArch64::ANDXri: NewOpc = AArch64::ANDSXri; break;
899 // Scan forward for the use of NZCV.
900 // When checking against MI: if it's a conditional code requires
901 // checking of V bit, then this is not safe to do.
902 // It is safe to remove CmpInstr if NZCV is redefined or killed.
903 // If we are done with the basic block, we need to check whether NZCV is
906 for (MachineBasicBlock::iterator I = CmpInstr,
907 E = CmpInstr->getParent()->end();
908 !IsSafe && ++I != E;) {
909 const MachineInstr &Instr = *I;
910 for (unsigned IO = 0, EO = Instr.getNumOperands(); !IsSafe && IO != EO;
912 const MachineOperand &MO = Instr.getOperand(IO);
913 if (MO.isRegMask() && MO.clobbersPhysReg(AArch64::NZCV)) {
917 if (!MO.isReg() || MO.getReg() != AArch64::NZCV)
924 // Decode the condition code.
925 unsigned Opc = Instr.getOpcode();
926 AArch64CC::CondCode CC;
931 CC = (AArch64CC::CondCode)Instr.getOperand(IO - 2).getImm();
933 case AArch64::CSINVWr:
934 case AArch64::CSINVXr:
935 case AArch64::CSINCWr:
936 case AArch64::CSINCXr:
937 case AArch64::CSELWr:
938 case AArch64::CSELXr:
939 case AArch64::CSNEGWr:
940 case AArch64::CSNEGXr:
941 case AArch64::FCSELSrrr:
942 case AArch64::FCSELDrrr:
943 CC = (AArch64CC::CondCode)Instr.getOperand(IO - 1).getImm();
947 // It is not safe to remove Compare instruction if Overflow(V) is used.
950 // NZCV can be used multiple times, we should continue.
963 // If NZCV is not killed nor re-defined, we should check whether it is
964 // live-out. If it is live-out, do not optimize.
966 MachineBasicBlock *ParentBlock = CmpInstr->getParent();
967 for (auto *MBB : ParentBlock->successors())
968 if (MBB->isLiveIn(AArch64::NZCV))
972 // Update the instruction to set NZCV.
973 MI->setDesc(get(NewOpc));
974 CmpInstr->eraseFromParent();
975 bool succeeded = UpdateOperandRegClass(MI);
977 assert(succeeded && "Some operands reg class are incompatible!");
978 MI->addRegisterDefined(AArch64::NZCV, TRI);
983 AArch64InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
984 if (MI->getOpcode() != TargetOpcode::LOAD_STACK_GUARD)
987 MachineBasicBlock &MBB = *MI->getParent();
988 DebugLoc DL = MI->getDebugLoc();
989 unsigned Reg = MI->getOperand(0).getReg();
990 const GlobalValue *GV =
991 cast<GlobalValue>((*MI->memoperands_begin())->getValue());
992 const TargetMachine &TM = MBB.getParent()->getTarget();
993 unsigned char OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
994 const unsigned char MO_NC = AArch64II::MO_NC;
996 if ((OpFlags & AArch64II::MO_GOT) != 0) {
997 BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
998 .addGlobalAddress(GV, 0, AArch64II::MO_GOT);
999 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1000 .addReg(Reg, RegState::Kill).addImm(0)
1001 .addMemOperand(*MI->memoperands_begin());
1002 } else if (TM.getCodeModel() == CodeModel::Large) {
1003 BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
1004 .addGlobalAddress(GV, 0, AArch64II::MO_G3).addImm(48);
1005 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1006 .addReg(Reg, RegState::Kill)
1007 .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC).addImm(32);
1008 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1009 .addReg(Reg, RegState::Kill)
1010 .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC).addImm(16);
1011 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1012 .addReg(Reg, RegState::Kill)
1013 .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC).addImm(0);
1014 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1015 .addReg(Reg, RegState::Kill).addImm(0)
1016 .addMemOperand(*MI->memoperands_begin());
1018 BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
1019 .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
1020 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
1021 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1022 .addReg(Reg, RegState::Kill)
1023 .addGlobalAddress(GV, 0, LoFlags)
1024 .addMemOperand(*MI->memoperands_begin());
1032 /// Return true if this is this instruction has a non-zero immediate
1033 bool AArch64InstrInfo::hasShiftedReg(const MachineInstr *MI) const {
1034 switch (MI->getOpcode()) {
1037 case AArch64::ADDSWrs:
1038 case AArch64::ADDSXrs:
1039 case AArch64::ADDWrs:
1040 case AArch64::ADDXrs:
1041 case AArch64::ANDSWrs:
1042 case AArch64::ANDSXrs:
1043 case AArch64::ANDWrs:
1044 case AArch64::ANDXrs:
1045 case AArch64::BICSWrs:
1046 case AArch64::BICSXrs:
1047 case AArch64::BICWrs:
1048 case AArch64::BICXrs:
1049 case AArch64::CRC32Brr:
1050 case AArch64::CRC32CBrr:
1051 case AArch64::CRC32CHrr:
1052 case AArch64::CRC32CWrr:
1053 case AArch64::CRC32CXrr:
1054 case AArch64::CRC32Hrr:
1055 case AArch64::CRC32Wrr:
1056 case AArch64::CRC32Xrr:
1057 case AArch64::EONWrs:
1058 case AArch64::EONXrs:
1059 case AArch64::EORWrs:
1060 case AArch64::EORXrs:
1061 case AArch64::ORNWrs:
1062 case AArch64::ORNXrs:
1063 case AArch64::ORRWrs:
1064 case AArch64::ORRXrs:
1065 case AArch64::SUBSWrs:
1066 case AArch64::SUBSXrs:
1067 case AArch64::SUBWrs:
1068 case AArch64::SUBXrs:
1069 if (MI->getOperand(3).isImm()) {
1070 unsigned val = MI->getOperand(3).getImm();
1078 /// Return true if this is this instruction has a non-zero immediate
1079 bool AArch64InstrInfo::hasExtendedReg(const MachineInstr *MI) const {
1080 switch (MI->getOpcode()) {
1083 case AArch64::ADDSWrx:
1084 case AArch64::ADDSXrx:
1085 case AArch64::ADDSXrx64:
1086 case AArch64::ADDWrx:
1087 case AArch64::ADDXrx:
1088 case AArch64::ADDXrx64:
1089 case AArch64::SUBSWrx:
1090 case AArch64::SUBSXrx:
1091 case AArch64::SUBSXrx64:
1092 case AArch64::SUBWrx:
1093 case AArch64::SUBXrx:
1094 case AArch64::SUBXrx64:
1095 if (MI->getOperand(3).isImm()) {
1096 unsigned val = MI->getOperand(3).getImm();
1105 // Return true if this instruction simply sets its single destination register
1106 // to zero. This is equivalent to a register rename of the zero-register.
1107 bool AArch64InstrInfo::isGPRZero(const MachineInstr *MI) const {
1108 switch (MI->getOpcode()) {
1111 case AArch64::MOVZWi:
1112 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
1113 if (MI->getOperand(1).isImm() && MI->getOperand(1).getImm() == 0) {
1114 assert(MI->getDesc().getNumOperands() == 3 &&
1115 MI->getOperand(2).getImm() == 0 && "invalid MOVZi operands");
1119 case AArch64::ANDWri: // and Rd, Rzr, #imm
1120 return MI->getOperand(1).getReg() == AArch64::WZR;
1121 case AArch64::ANDXri:
1122 return MI->getOperand(1).getReg() == AArch64::XZR;
1123 case TargetOpcode::COPY:
1124 return MI->getOperand(1).getReg() == AArch64::WZR;
1129 // Return true if this instruction simply renames a general register without
1131 bool AArch64InstrInfo::isGPRCopy(const MachineInstr *MI) const {
1132 switch (MI->getOpcode()) {
1135 case TargetOpcode::COPY: {
1136 // GPR32 copies will by lowered to ORRXrs
1137 unsigned DstReg = MI->getOperand(0).getReg();
1138 return (AArch64::GPR32RegClass.contains(DstReg) ||
1139 AArch64::GPR64RegClass.contains(DstReg));
1141 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
1142 if (MI->getOperand(1).getReg() == AArch64::XZR) {
1143 assert(MI->getDesc().getNumOperands() == 4 &&
1144 MI->getOperand(3).getImm() == 0 && "invalid ORRrs operands");
1148 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
1149 if (MI->getOperand(2).getImm() == 0) {
1150 assert(MI->getDesc().getNumOperands() == 4 &&
1151 MI->getOperand(3).getImm() == 0 && "invalid ADDXri operands");
1159 // Return true if this instruction simply renames a general register without
1161 bool AArch64InstrInfo::isFPRCopy(const MachineInstr *MI) const {
1162 switch (MI->getOpcode()) {
1165 case TargetOpcode::COPY: {
1166 // FPR64 copies will by lowered to ORR.16b
1167 unsigned DstReg = MI->getOperand(0).getReg();
1168 return (AArch64::FPR64RegClass.contains(DstReg) ||
1169 AArch64::FPR128RegClass.contains(DstReg));
1171 case AArch64::ORRv16i8:
1172 if (MI->getOperand(1).getReg() == MI->getOperand(2).getReg()) {
1173 assert(MI->getDesc().getNumOperands() == 3 && MI->getOperand(0).isReg() &&
1174 "invalid ORRv16i8 operands");
1182 unsigned AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
1183 int &FrameIndex) const {
1184 switch (MI->getOpcode()) {
1187 case AArch64::LDRWui:
1188 case AArch64::LDRXui:
1189 case AArch64::LDRBui:
1190 case AArch64::LDRHui:
1191 case AArch64::LDRSui:
1192 case AArch64::LDRDui:
1193 case AArch64::LDRQui:
1194 if (MI->getOperand(0).getSubReg() == 0 && MI->getOperand(1).isFI() &&
1195 MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0) {
1196 FrameIndex = MI->getOperand(1).getIndex();
1197 return MI->getOperand(0).getReg();
1205 unsigned AArch64InstrInfo::isStoreToStackSlot(const MachineInstr *MI,
1206 int &FrameIndex) const {
1207 switch (MI->getOpcode()) {
1210 case AArch64::STRWui:
1211 case AArch64::STRXui:
1212 case AArch64::STRBui:
1213 case AArch64::STRHui:
1214 case AArch64::STRSui:
1215 case AArch64::STRDui:
1216 case AArch64::STRQui:
1217 if (MI->getOperand(0).getSubReg() == 0 && MI->getOperand(1).isFI() &&
1218 MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0) {
1219 FrameIndex = MI->getOperand(1).getIndex();
1220 return MI->getOperand(0).getReg();
1227 /// Return true if this is load/store scales or extends its register offset.
1228 /// This refers to scaling a dynamic index as opposed to scaled immediates.
1229 /// MI should be a memory op that allows scaled addressing.
1230 bool AArch64InstrInfo::isScaledAddr(const MachineInstr *MI) const {
1231 switch (MI->getOpcode()) {
1234 case AArch64::LDRBBroW:
1235 case AArch64::LDRBroW:
1236 case AArch64::LDRDroW:
1237 case AArch64::LDRHHroW:
1238 case AArch64::LDRHroW:
1239 case AArch64::LDRQroW:
1240 case AArch64::LDRSBWroW:
1241 case AArch64::LDRSBXroW:
1242 case AArch64::LDRSHWroW:
1243 case AArch64::LDRSHXroW:
1244 case AArch64::LDRSWroW:
1245 case AArch64::LDRSroW:
1246 case AArch64::LDRWroW:
1247 case AArch64::LDRXroW:
1248 case AArch64::STRBBroW:
1249 case AArch64::STRBroW:
1250 case AArch64::STRDroW:
1251 case AArch64::STRHHroW:
1252 case AArch64::STRHroW:
1253 case AArch64::STRQroW:
1254 case AArch64::STRSroW:
1255 case AArch64::STRWroW:
1256 case AArch64::STRXroW:
1257 case AArch64::LDRBBroX:
1258 case AArch64::LDRBroX:
1259 case AArch64::LDRDroX:
1260 case AArch64::LDRHHroX:
1261 case AArch64::LDRHroX:
1262 case AArch64::LDRQroX:
1263 case AArch64::LDRSBWroX:
1264 case AArch64::LDRSBXroX:
1265 case AArch64::LDRSHWroX:
1266 case AArch64::LDRSHXroX:
1267 case AArch64::LDRSWroX:
1268 case AArch64::LDRSroX:
1269 case AArch64::LDRWroX:
1270 case AArch64::LDRXroX:
1271 case AArch64::STRBBroX:
1272 case AArch64::STRBroX:
1273 case AArch64::STRDroX:
1274 case AArch64::STRHHroX:
1275 case AArch64::STRHroX:
1276 case AArch64::STRQroX:
1277 case AArch64::STRSroX:
1278 case AArch64::STRWroX:
1279 case AArch64::STRXroX:
1281 unsigned Val = MI->getOperand(3).getImm();
1282 AArch64_AM::ShiftExtendType ExtType = AArch64_AM::getMemExtendType(Val);
1283 return (ExtType != AArch64_AM::UXTX) || AArch64_AM::getMemDoShift(Val);
1288 /// Check all MachineMemOperands for a hint to suppress pairing.
1289 bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr *MI) const {
1290 assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits) &&
1291 "Too many target MO flags");
1292 for (auto *MM : MI->memoperands()) {
1293 if (MM->getFlags() &
1294 (MOSuppressPair << MachineMemOperand::MOTargetStartBit)) {
1301 /// Set a flag on the first MachineMemOperand to suppress pairing.
1302 void AArch64InstrInfo::suppressLdStPair(MachineInstr *MI) const {
1303 if (MI->memoperands_empty())
1306 assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits) &&
1307 "Too many target MO flags");
1308 (*MI->memoperands_begin())
1309 ->setFlags(MOSuppressPair << MachineMemOperand::MOTargetStartBit);
1313 AArch64InstrInfo::getLdStBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
1315 const TargetRegisterInfo *TRI) const {
1316 switch (LdSt->getOpcode()) {
1319 case AArch64::STRSui:
1320 case AArch64::STRDui:
1321 case AArch64::STRQui:
1322 case AArch64::STRXui:
1323 case AArch64::STRWui:
1324 case AArch64::LDRSui:
1325 case AArch64::LDRDui:
1326 case AArch64::LDRQui:
1327 case AArch64::LDRXui:
1328 case AArch64::LDRWui:
1329 if (!LdSt->getOperand(1).isReg() || !LdSt->getOperand(2).isImm())
1331 BaseReg = LdSt->getOperand(1).getReg();
1332 MachineFunction &MF = *LdSt->getParent()->getParent();
1333 unsigned Width = getRegClass(LdSt->getDesc(), 0, TRI, MF)->getSize();
1334 Offset = LdSt->getOperand(2).getImm() * Width;
1339 bool AArch64InstrInfo::getLdStBaseRegImmOfsWidth(
1340 MachineInstr *LdSt, unsigned &BaseReg, int &Offset, int &Width,
1341 const TargetRegisterInfo *TRI) const {
1342 // Handle only loads/stores with base register followed by immediate offset.
1343 if (LdSt->getNumOperands() != 3)
1345 if (!LdSt->getOperand(1).isReg() || !LdSt->getOperand(2).isImm())
1348 // Offset is calculated as the immediate operand multiplied by the scaling factor.
1349 // Unscaled instructions have scaling factor set to 1.
1351 switch (LdSt->getOpcode()) {
1354 case AArch64::LDURQi:
1355 case AArch64::STURQi:
1359 case AArch64::LDURXi:
1360 case AArch64::LDURDi:
1361 case AArch64::STURXi:
1362 case AArch64::STURDi:
1366 case AArch64::LDURWi:
1367 case AArch64::LDURSi:
1368 case AArch64::LDURSWi:
1369 case AArch64::STURWi:
1370 case AArch64::STURSi:
1374 case AArch64::LDURHi:
1375 case AArch64::LDURHHi:
1376 case AArch64::LDURSHXi:
1377 case AArch64::LDURSHWi:
1378 case AArch64::STURHi:
1379 case AArch64::STURHHi:
1383 case AArch64::LDURBi:
1384 case AArch64::LDURBBi:
1385 case AArch64::LDURSBXi:
1386 case AArch64::LDURSBWi:
1387 case AArch64::STURBi:
1388 case AArch64::STURBBi:
1392 case AArch64::LDRXui:
1393 case AArch64::STRXui:
1396 case AArch64::LDRWui:
1397 case AArch64::STRWui:
1400 case AArch64::LDRBui:
1401 case AArch64::STRBui:
1404 case AArch64::LDRHui:
1405 case AArch64::STRHui:
1408 case AArch64::LDRSui:
1409 case AArch64::STRSui:
1412 case AArch64::LDRDui:
1413 case AArch64::STRDui:
1416 case AArch64::LDRQui:
1417 case AArch64::STRQui:
1420 case AArch64::LDRBBui:
1421 case AArch64::STRBBui:
1424 case AArch64::LDRHHui:
1425 case AArch64::STRHHui:
1430 BaseReg = LdSt->getOperand(1).getReg();
1431 Offset = LdSt->getOperand(2).getImm() * Scale;
1435 /// Detect opportunities for ldp/stp formation.
1437 /// Only called for LdSt for which getLdStBaseRegImmOfs returns true.
1438 bool AArch64InstrInfo::shouldClusterLoads(MachineInstr *FirstLdSt,
1439 MachineInstr *SecondLdSt,
1440 unsigned NumLoads) const {
1441 // Only cluster up to a single pair.
1444 if (FirstLdSt->getOpcode() != SecondLdSt->getOpcode())
1446 // getLdStBaseRegImmOfs guarantees that oper 2 isImm.
1447 unsigned Ofs1 = FirstLdSt->getOperand(2).getImm();
1448 // Allow 6 bits of positive range.
1451 // The caller should already have ordered First/SecondLdSt by offset.
1452 unsigned Ofs2 = SecondLdSt->getOperand(2).getImm();
1453 return Ofs1 + 1 == Ofs2;
1456 bool AArch64InstrInfo::shouldScheduleAdjacent(MachineInstr *First,
1457 MachineInstr *Second) const {
1458 // Cyclone can fuse CMN, CMP followed by Bcc.
1460 // FIXME: B0 can also fuse:
1461 // AND, BIC, ORN, ORR, or EOR (optional S) followed by Bcc or CBZ or CBNZ.
1462 if (Second->getOpcode() != AArch64::Bcc)
1464 switch (First->getOpcode()) {
1467 case AArch64::SUBSWri:
1468 case AArch64::ADDSWri:
1469 case AArch64::ANDSWri:
1470 case AArch64::SUBSXri:
1471 case AArch64::ADDSXri:
1472 case AArch64::ANDSXri:
1477 MachineInstr *AArch64InstrInfo::emitFrameIndexDebugValue(
1478 MachineFunction &MF, int FrameIx, uint64_t Offset, const MDNode *Var,
1479 const MDNode *Expr, DebugLoc DL) const {
1480 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AArch64::DBG_VALUE))
1481 .addFrameIndex(FrameIx)
1489 static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
1490 unsigned Reg, unsigned SubIdx,
1492 const TargetRegisterInfo *TRI) {
1494 return MIB.addReg(Reg, State);
1496 if (TargetRegisterInfo::isPhysicalRegister(Reg))
1497 return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
1498 return MIB.addReg(Reg, State, SubIdx);
1501 static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
1503 // We really want the positive remainder mod 32 here, that happens to be
1504 // easily obtainable with a mask.
1505 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
1508 void AArch64InstrInfo::copyPhysRegTuple(
1509 MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL,
1510 unsigned DestReg, unsigned SrcReg, bool KillSrc, unsigned Opcode,
1511 llvm::ArrayRef<unsigned> Indices) const {
1512 assert(Subtarget.hasNEON() &&
1513 "Unexpected register copy without NEON");
1514 const TargetRegisterInfo *TRI = &getRegisterInfo();
1515 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
1516 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
1517 unsigned NumRegs = Indices.size();
1519 int SubReg = 0, End = NumRegs, Incr = 1;
1520 if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
1521 SubReg = NumRegs - 1;
1526 for (; SubReg != End; SubReg += Incr) {
1527 const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
1528 AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
1529 AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
1530 AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
1534 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1535 MachineBasicBlock::iterator I, DebugLoc DL,
1536 unsigned DestReg, unsigned SrcReg,
1537 bool KillSrc) const {
1538 if (AArch64::GPR32spRegClass.contains(DestReg) &&
1539 (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) {
1540 const TargetRegisterInfo *TRI = &getRegisterInfo();
1542 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
1543 // If either operand is WSP, expand to ADD #0.
1544 if (Subtarget.hasZeroCycleRegMove()) {
1545 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
1546 unsigned DestRegX = TRI->getMatchingSuperReg(DestReg, AArch64::sub_32,
1547 &AArch64::GPR64spRegClass);
1548 unsigned SrcRegX = TRI->getMatchingSuperReg(SrcReg, AArch64::sub_32,
1549 &AArch64::GPR64spRegClass);
1550 // This instruction is reading and writing X registers. This may upset
1551 // the register scavenger and machine verifier, so we need to indicate
1552 // that we are reading an undefined value from SrcRegX, but a proper
1553 // value from SrcReg.
1554 BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
1555 .addReg(SrcRegX, RegState::Undef)
1557 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
1558 .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
1560 BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
1561 .addReg(SrcReg, getKillRegState(KillSrc))
1563 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1565 } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroing()) {
1566 BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg).addImm(0).addImm(
1567 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1569 if (Subtarget.hasZeroCycleRegMove()) {
1570 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
1571 unsigned DestRegX = TRI->getMatchingSuperReg(DestReg, AArch64::sub_32,
1572 &AArch64::GPR64spRegClass);
1573 unsigned SrcRegX = TRI->getMatchingSuperReg(SrcReg, AArch64::sub_32,
1574 &AArch64::GPR64spRegClass);
1575 // This instruction is reading and writing X registers. This may upset
1576 // the register scavenger and machine verifier, so we need to indicate
1577 // that we are reading an undefined value from SrcRegX, but a proper
1578 // value from SrcReg.
1579 BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
1580 .addReg(AArch64::XZR)
1581 .addReg(SrcRegX, RegState::Undef)
1582 .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
1584 // Otherwise, expand to ORR WZR.
1585 BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
1586 .addReg(AArch64::WZR)
1587 .addReg(SrcReg, getKillRegState(KillSrc));
1593 if (AArch64::GPR64spRegClass.contains(DestReg) &&
1594 (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
1595 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
1596 // If either operand is SP, expand to ADD #0.
1597 BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
1598 .addReg(SrcReg, getKillRegState(KillSrc))
1600 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1601 } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroing()) {
1602 BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg).addImm(0).addImm(
1603 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1605 // Otherwise, expand to ORR XZR.
1606 BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
1607 .addReg(AArch64::XZR)
1608 .addReg(SrcReg, getKillRegState(KillSrc));
1613 // Copy a DDDD register quad by copying the individual sub-registers.
1614 if (AArch64::DDDDRegClass.contains(DestReg) &&
1615 AArch64::DDDDRegClass.contains(SrcReg)) {
1616 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1,
1617 AArch64::dsub2, AArch64::dsub3 };
1618 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1623 // Copy a DDD register triple by copying the individual sub-registers.
1624 if (AArch64::DDDRegClass.contains(DestReg) &&
1625 AArch64::DDDRegClass.contains(SrcReg)) {
1626 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1,
1628 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1633 // Copy a DD register pair by copying the individual sub-registers.
1634 if (AArch64::DDRegClass.contains(DestReg) &&
1635 AArch64::DDRegClass.contains(SrcReg)) {
1636 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1 };
1637 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1642 // Copy a QQQQ register quad by copying the individual sub-registers.
1643 if (AArch64::QQQQRegClass.contains(DestReg) &&
1644 AArch64::QQQQRegClass.contains(SrcReg)) {
1645 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1,
1646 AArch64::qsub2, AArch64::qsub3 };
1647 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1652 // Copy a QQQ register triple by copying the individual sub-registers.
1653 if (AArch64::QQQRegClass.contains(DestReg) &&
1654 AArch64::QQQRegClass.contains(SrcReg)) {
1655 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1,
1657 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1662 // Copy a QQ register pair by copying the individual sub-registers.
1663 if (AArch64::QQRegClass.contains(DestReg) &&
1664 AArch64::QQRegClass.contains(SrcReg)) {
1665 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1 };
1666 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1671 if (AArch64::FPR128RegClass.contains(DestReg) &&
1672 AArch64::FPR128RegClass.contains(SrcReg)) {
1673 if(Subtarget.hasNEON()) {
1674 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1676 .addReg(SrcReg, getKillRegState(KillSrc));
1678 BuildMI(MBB, I, DL, get(AArch64::STRQpre))
1679 .addReg(AArch64::SP, RegState::Define)
1680 .addReg(SrcReg, getKillRegState(KillSrc))
1681 .addReg(AArch64::SP)
1683 BuildMI(MBB, I, DL, get(AArch64::LDRQpre))
1684 .addReg(AArch64::SP, RegState::Define)
1685 .addReg(DestReg, RegState::Define)
1686 .addReg(AArch64::SP)
1692 if (AArch64::FPR64RegClass.contains(DestReg) &&
1693 AArch64::FPR64RegClass.contains(SrcReg)) {
1694 if(Subtarget.hasNEON()) {
1695 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
1696 &AArch64::FPR128RegClass);
1697 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
1698 &AArch64::FPR128RegClass);
1699 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1701 .addReg(SrcReg, getKillRegState(KillSrc));
1703 BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
1704 .addReg(SrcReg, getKillRegState(KillSrc));
1709 if (AArch64::FPR32RegClass.contains(DestReg) &&
1710 AArch64::FPR32RegClass.contains(SrcReg)) {
1711 if(Subtarget.hasNEON()) {
1712 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
1713 &AArch64::FPR128RegClass);
1714 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
1715 &AArch64::FPR128RegClass);
1716 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1718 .addReg(SrcReg, getKillRegState(KillSrc));
1720 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1721 .addReg(SrcReg, getKillRegState(KillSrc));
1726 if (AArch64::FPR16RegClass.contains(DestReg) &&
1727 AArch64::FPR16RegClass.contains(SrcReg)) {
1728 if(Subtarget.hasNEON()) {
1729 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
1730 &AArch64::FPR128RegClass);
1731 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
1732 &AArch64::FPR128RegClass);
1733 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1735 .addReg(SrcReg, getKillRegState(KillSrc));
1737 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
1738 &AArch64::FPR32RegClass);
1739 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
1740 &AArch64::FPR32RegClass);
1741 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1742 .addReg(SrcReg, getKillRegState(KillSrc));
1747 if (AArch64::FPR8RegClass.contains(DestReg) &&
1748 AArch64::FPR8RegClass.contains(SrcReg)) {
1749 if(Subtarget.hasNEON()) {
1750 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
1751 &AArch64::FPR128RegClass);
1752 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
1753 &AArch64::FPR128RegClass);
1754 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1756 .addReg(SrcReg, getKillRegState(KillSrc));
1758 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
1759 &AArch64::FPR32RegClass);
1760 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
1761 &AArch64::FPR32RegClass);
1762 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1763 .addReg(SrcReg, getKillRegState(KillSrc));
1768 // Copies between GPR64 and FPR64.
1769 if (AArch64::FPR64RegClass.contains(DestReg) &&
1770 AArch64::GPR64RegClass.contains(SrcReg)) {
1771 BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
1772 .addReg(SrcReg, getKillRegState(KillSrc));
1775 if (AArch64::GPR64RegClass.contains(DestReg) &&
1776 AArch64::FPR64RegClass.contains(SrcReg)) {
1777 BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
1778 .addReg(SrcReg, getKillRegState(KillSrc));
1781 // Copies between GPR32 and FPR32.
1782 if (AArch64::FPR32RegClass.contains(DestReg) &&
1783 AArch64::GPR32RegClass.contains(SrcReg)) {
1784 BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
1785 .addReg(SrcReg, getKillRegState(KillSrc));
1788 if (AArch64::GPR32RegClass.contains(DestReg) &&
1789 AArch64::FPR32RegClass.contains(SrcReg)) {
1790 BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
1791 .addReg(SrcReg, getKillRegState(KillSrc));
1795 if (DestReg == AArch64::NZCV) {
1796 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
1797 BuildMI(MBB, I, DL, get(AArch64::MSR))
1798 .addImm(AArch64SysReg::NZCV)
1799 .addReg(SrcReg, getKillRegState(KillSrc))
1800 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
1804 if (SrcReg == AArch64::NZCV) {
1805 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
1806 BuildMI(MBB, I, DL, get(AArch64::MRS))
1808 .addImm(AArch64SysReg::NZCV)
1809 .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
1813 llvm_unreachable("unimplemented reg-to-reg copy");
1816 void AArch64InstrInfo::storeRegToStackSlot(
1817 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned SrcReg,
1818 bool isKill, int FI, const TargetRegisterClass *RC,
1819 const TargetRegisterInfo *TRI) const {
1821 if (MBBI != MBB.end())
1822 DL = MBBI->getDebugLoc();
1823 MachineFunction &MF = *MBB.getParent();
1824 MachineFrameInfo &MFI = *MF.getFrameInfo();
1825 unsigned Align = MFI.getObjectAlignment(FI);
1827 MachinePointerInfo PtrInfo(PseudoSourceValue::getFixedStack(FI));
1828 MachineMemOperand *MMO = MF.getMachineMemOperand(
1829 PtrInfo, MachineMemOperand::MOStore, MFI.getObjectSize(FI), Align);
1832 switch (RC->getSize()) {
1834 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
1835 Opc = AArch64::STRBui;
1838 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
1839 Opc = AArch64::STRHui;
1842 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1843 Opc = AArch64::STRWui;
1844 if (TargetRegisterInfo::isVirtualRegister(SrcReg))
1845 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
1847 assert(SrcReg != AArch64::WSP);
1848 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
1849 Opc = AArch64::STRSui;
1852 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
1853 Opc = AArch64::STRXui;
1854 if (TargetRegisterInfo::isVirtualRegister(SrcReg))
1855 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
1857 assert(SrcReg != AArch64::SP);
1858 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
1859 Opc = AArch64::STRDui;
1862 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
1863 Opc = AArch64::STRQui;
1864 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
1865 assert(Subtarget.hasNEON() &&
1866 "Unexpected register store without NEON");
1867 Opc = AArch64::ST1Twov1d, Offset = false;
1871 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
1872 assert(Subtarget.hasNEON() &&
1873 "Unexpected register store without NEON");
1874 Opc = AArch64::ST1Threev1d, Offset = false;
1878 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
1879 assert(Subtarget.hasNEON() &&
1880 "Unexpected register store without NEON");
1881 Opc = AArch64::ST1Fourv1d, Offset = false;
1882 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
1883 assert(Subtarget.hasNEON() &&
1884 "Unexpected register store without NEON");
1885 Opc = AArch64::ST1Twov2d, Offset = false;
1889 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
1890 assert(Subtarget.hasNEON() &&
1891 "Unexpected register store without NEON");
1892 Opc = AArch64::ST1Threev2d, Offset = false;
1896 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
1897 assert(Subtarget.hasNEON() &&
1898 "Unexpected register store without NEON");
1899 Opc = AArch64::ST1Fourv2d, Offset = false;
1903 assert(Opc && "Unknown register class");
1905 const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DL, get(Opc))
1906 .addReg(SrcReg, getKillRegState(isKill))
1911 MI.addMemOperand(MMO);
1914 void AArch64InstrInfo::loadRegFromStackSlot(
1915 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned DestReg,
1916 int FI, const TargetRegisterClass *RC,
1917 const TargetRegisterInfo *TRI) const {
1919 if (MBBI != MBB.end())
1920 DL = MBBI->getDebugLoc();
1921 MachineFunction &MF = *MBB.getParent();
1922 MachineFrameInfo &MFI = *MF.getFrameInfo();
1923 unsigned Align = MFI.getObjectAlignment(FI);
1924 MachinePointerInfo PtrInfo(PseudoSourceValue::getFixedStack(FI));
1925 MachineMemOperand *MMO = MF.getMachineMemOperand(
1926 PtrInfo, MachineMemOperand::MOLoad, MFI.getObjectSize(FI), Align);
1930 switch (RC->getSize()) {
1932 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
1933 Opc = AArch64::LDRBui;
1936 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
1937 Opc = AArch64::LDRHui;
1940 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1941 Opc = AArch64::LDRWui;
1942 if (TargetRegisterInfo::isVirtualRegister(DestReg))
1943 MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
1945 assert(DestReg != AArch64::WSP);
1946 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
1947 Opc = AArch64::LDRSui;
1950 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
1951 Opc = AArch64::LDRXui;
1952 if (TargetRegisterInfo::isVirtualRegister(DestReg))
1953 MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
1955 assert(DestReg != AArch64::SP);
1956 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
1957 Opc = AArch64::LDRDui;
1960 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
1961 Opc = AArch64::LDRQui;
1962 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
1963 assert(Subtarget.hasNEON() &&
1964 "Unexpected register load without NEON");
1965 Opc = AArch64::LD1Twov1d, Offset = false;
1969 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
1970 assert(Subtarget.hasNEON() &&
1971 "Unexpected register load without NEON");
1972 Opc = AArch64::LD1Threev1d, Offset = false;
1976 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
1977 assert(Subtarget.hasNEON() &&
1978 "Unexpected register load without NEON");
1979 Opc = AArch64::LD1Fourv1d, Offset = false;
1980 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
1981 assert(Subtarget.hasNEON() &&
1982 "Unexpected register load without NEON");
1983 Opc = AArch64::LD1Twov2d, Offset = false;
1987 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
1988 assert(Subtarget.hasNEON() &&
1989 "Unexpected register load without NEON");
1990 Opc = AArch64::LD1Threev2d, Offset = false;
1994 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
1995 assert(Subtarget.hasNEON() &&
1996 "Unexpected register load without NEON");
1997 Opc = AArch64::LD1Fourv2d, Offset = false;
2001 assert(Opc && "Unknown register class");
2003 const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DL, get(Opc))
2004 .addReg(DestReg, getDefRegState(true))
2008 MI.addMemOperand(MMO);
2011 void llvm::emitFrameOffset(MachineBasicBlock &MBB,
2012 MachineBasicBlock::iterator MBBI, DebugLoc DL,
2013 unsigned DestReg, unsigned SrcReg, int Offset,
2014 const TargetInstrInfo *TII,
2015 MachineInstr::MIFlag Flag, bool SetNZCV) {
2016 if (DestReg == SrcReg && Offset == 0)
2019 bool isSub = Offset < 0;
2023 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
2024 // scratch register. If DestReg is a virtual register, use it as the
2025 // scratch register; otherwise, create a new virtual register (to be
2026 // replaced by the scavenger at the end of PEI). That case can be optimized
2027 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
2028 // register can be loaded with offset%8 and the add/sub can use an extending
2029 // instruction with LSL#3.
2030 // Currently the function handles any offsets but generates a poor sequence
2032 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
2036 Opc = isSub ? AArch64::SUBSXri : AArch64::ADDSXri;
2038 Opc = isSub ? AArch64::SUBXri : AArch64::ADDXri;
2039 const unsigned MaxEncoding = 0xfff;
2040 const unsigned ShiftSize = 12;
2041 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
2042 while (((unsigned)Offset) >= (1 << ShiftSize)) {
2044 if (((unsigned)Offset) > MaxEncodableValue) {
2045 ThisVal = MaxEncodableValue;
2047 ThisVal = Offset & MaxEncodableValue;
2049 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
2050 "Encoding cannot handle value that big");
2051 BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
2053 .addImm(ThisVal >> ShiftSize)
2054 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftSize))
2062 BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
2065 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
2069 MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
2071 ArrayRef<unsigned> Ops,
2072 int FrameIndex) const {
2073 // This is a bit of a hack. Consider this instruction:
2075 // %vreg0<def> = COPY %SP; GPR64all:%vreg0
2077 // We explicitly chose GPR64all for the virtual register so such a copy might
2078 // be eliminated by RegisterCoalescer. However, that may not be possible, and
2079 // %vreg0 may even spill. We can't spill %SP, and since it is in the GPR64all
2080 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
2082 // To prevent that, we are going to constrain the %vreg0 register class here.
2084 // <rdar://problem/11522048>
2087 unsigned DstReg = MI->getOperand(0).getReg();
2088 unsigned SrcReg = MI->getOperand(1).getReg();
2089 if (SrcReg == AArch64::SP &&
2090 TargetRegisterInfo::isVirtualRegister(DstReg)) {
2091 MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
2094 if (DstReg == AArch64::SP &&
2095 TargetRegisterInfo::isVirtualRegister(SrcReg)) {
2096 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
2105 int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI, int &Offset,
2106 bool *OutUseUnscaledOp,
2107 unsigned *OutUnscaledOp,
2108 int *EmittableOffset) {
2110 bool IsSigned = false;
2111 // The ImmIdx should be changed case by case if it is not 2.
2112 unsigned ImmIdx = 2;
2113 unsigned UnscaledOp = 0;
2114 // Set output values in case of early exit.
2115 if (EmittableOffset)
2116 *EmittableOffset = 0;
2117 if (OutUseUnscaledOp)
2118 *OutUseUnscaledOp = false;
2121 switch (MI.getOpcode()) {
2123 llvm_unreachable("unhandled opcode in rewriteAArch64FrameIndex");
2124 // Vector spills/fills can't take an immediate offset.
2125 case AArch64::LD1Twov2d:
2126 case AArch64::LD1Threev2d:
2127 case AArch64::LD1Fourv2d:
2128 case AArch64::LD1Twov1d:
2129 case AArch64::LD1Threev1d:
2130 case AArch64::LD1Fourv1d:
2131 case AArch64::ST1Twov2d:
2132 case AArch64::ST1Threev2d:
2133 case AArch64::ST1Fourv2d:
2134 case AArch64::ST1Twov1d:
2135 case AArch64::ST1Threev1d:
2136 case AArch64::ST1Fourv1d:
2137 return AArch64FrameOffsetCannotUpdate;
2138 case AArch64::PRFMui:
2140 UnscaledOp = AArch64::PRFUMi;
2142 case AArch64::LDRXui:
2144 UnscaledOp = AArch64::LDURXi;
2146 case AArch64::LDRWui:
2148 UnscaledOp = AArch64::LDURWi;
2150 case AArch64::LDRBui:
2152 UnscaledOp = AArch64::LDURBi;
2154 case AArch64::LDRHui:
2156 UnscaledOp = AArch64::LDURHi;
2158 case AArch64::LDRSui:
2160 UnscaledOp = AArch64::LDURSi;
2162 case AArch64::LDRDui:
2164 UnscaledOp = AArch64::LDURDi;
2166 case AArch64::LDRQui:
2168 UnscaledOp = AArch64::LDURQi;
2170 case AArch64::LDRBBui:
2172 UnscaledOp = AArch64::LDURBBi;
2174 case AArch64::LDRHHui:
2176 UnscaledOp = AArch64::LDURHHi;
2178 case AArch64::LDRSBXui:
2180 UnscaledOp = AArch64::LDURSBXi;
2182 case AArch64::LDRSBWui:
2184 UnscaledOp = AArch64::LDURSBWi;
2186 case AArch64::LDRSHXui:
2188 UnscaledOp = AArch64::LDURSHXi;
2190 case AArch64::LDRSHWui:
2192 UnscaledOp = AArch64::LDURSHWi;
2194 case AArch64::LDRSWui:
2196 UnscaledOp = AArch64::LDURSWi;
2199 case AArch64::STRXui:
2201 UnscaledOp = AArch64::STURXi;
2203 case AArch64::STRWui:
2205 UnscaledOp = AArch64::STURWi;
2207 case AArch64::STRBui:
2209 UnscaledOp = AArch64::STURBi;
2211 case AArch64::STRHui:
2213 UnscaledOp = AArch64::STURHi;
2215 case AArch64::STRSui:
2217 UnscaledOp = AArch64::STURSi;
2219 case AArch64::STRDui:
2221 UnscaledOp = AArch64::STURDi;
2223 case AArch64::STRQui:
2225 UnscaledOp = AArch64::STURQi;
2227 case AArch64::STRBBui:
2229 UnscaledOp = AArch64::STURBBi;
2231 case AArch64::STRHHui:
2233 UnscaledOp = AArch64::STURHHi;
2236 case AArch64::LDPXi:
2237 case AArch64::LDPDi:
2238 case AArch64::STPXi:
2239 case AArch64::STPDi:
2243 case AArch64::LDPQi:
2244 case AArch64::STPQi:
2248 case AArch64::LDPWi:
2249 case AArch64::LDPSi:
2250 case AArch64::STPWi:
2251 case AArch64::STPSi:
2256 case AArch64::LDURXi:
2257 case AArch64::LDURWi:
2258 case AArch64::LDURBi:
2259 case AArch64::LDURHi:
2260 case AArch64::LDURSi:
2261 case AArch64::LDURDi:
2262 case AArch64::LDURQi:
2263 case AArch64::LDURHHi:
2264 case AArch64::LDURBBi:
2265 case AArch64::LDURSBXi:
2266 case AArch64::LDURSBWi:
2267 case AArch64::LDURSHXi:
2268 case AArch64::LDURSHWi:
2269 case AArch64::LDURSWi:
2270 case AArch64::STURXi:
2271 case AArch64::STURWi:
2272 case AArch64::STURBi:
2273 case AArch64::STURHi:
2274 case AArch64::STURSi:
2275 case AArch64::STURDi:
2276 case AArch64::STURQi:
2277 case AArch64::STURBBi:
2278 case AArch64::STURHHi:
2283 Offset += MI.getOperand(ImmIdx).getImm() * Scale;
2285 bool useUnscaledOp = false;
2286 // If the offset doesn't match the scale, we rewrite the instruction to
2287 // use the unscaled instruction instead. Likewise, if we have a negative
2288 // offset (and have an unscaled op to use).
2289 if ((Offset & (Scale - 1)) != 0 || (Offset < 0 && UnscaledOp != 0))
2290 useUnscaledOp = true;
2292 // Use an unscaled addressing mode if the instruction has a negative offset
2293 // (or if the instruction is already using an unscaled addressing mode).
2296 // ldp/stp instructions.
2299 } else if (UnscaledOp == 0 || useUnscaledOp) {
2309 // Attempt to fold address computation.
2310 int MaxOff = (1 << (MaskBits - IsSigned)) - 1;
2311 int MinOff = (IsSigned ? (-MaxOff - 1) : 0);
2312 if (Offset >= MinOff && Offset <= MaxOff) {
2313 if (EmittableOffset)
2314 *EmittableOffset = Offset;
2317 int NewOff = Offset < 0 ? MinOff : MaxOff;
2318 if (EmittableOffset)
2319 *EmittableOffset = NewOff;
2320 Offset = (Offset - NewOff) * Scale;
2322 if (OutUseUnscaledOp)
2323 *OutUseUnscaledOp = useUnscaledOp;
2325 *OutUnscaledOp = UnscaledOp;
2326 return AArch64FrameOffsetCanUpdate |
2327 (Offset == 0 ? AArch64FrameOffsetIsLegal : 0);
2330 bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2331 unsigned FrameReg, int &Offset,
2332 const AArch64InstrInfo *TII) {
2333 unsigned Opcode = MI.getOpcode();
2334 unsigned ImmIdx = FrameRegIdx + 1;
2336 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
2337 Offset += MI.getOperand(ImmIdx).getImm();
2338 emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
2339 MI.getOperand(0).getReg(), FrameReg, Offset, TII,
2340 MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
2341 MI.eraseFromParent();
2347 unsigned UnscaledOp;
2349 int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
2350 &UnscaledOp, &NewOffset);
2351 if (Status & AArch64FrameOffsetCanUpdate) {
2352 if (Status & AArch64FrameOffsetIsLegal)
2353 // Replace the FrameIndex with FrameReg.
2354 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2356 MI.setDesc(TII->get(UnscaledOp));
2358 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
2365 void AArch64InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
2366 NopInst.setOpcode(AArch64::HINT);
2367 NopInst.addOperand(MCOperand::createImm(0));
2369 /// useMachineCombiner - return true when a target supports MachineCombiner
2370 bool AArch64InstrInfo::useMachineCombiner() const {
2371 // AArch64 supports the combiner
2375 // True when Opc sets flag
2376 static bool isCombineInstrSettingFlag(unsigned Opc) {
2378 case AArch64::ADDSWrr:
2379 case AArch64::ADDSWri:
2380 case AArch64::ADDSXrr:
2381 case AArch64::ADDSXri:
2382 case AArch64::SUBSWrr:
2383 case AArch64::SUBSXrr:
2384 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2385 case AArch64::SUBSWri:
2386 case AArch64::SUBSXri:
2394 // 32b Opcodes that can be combined with a MUL
2395 static bool isCombineInstrCandidate32(unsigned Opc) {
2397 case AArch64::ADDWrr:
2398 case AArch64::ADDWri:
2399 case AArch64::SUBWrr:
2400 case AArch64::ADDSWrr:
2401 case AArch64::ADDSWri:
2402 case AArch64::SUBSWrr:
2403 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2404 case AArch64::SUBWri:
2405 case AArch64::SUBSWri:
2413 // 64b Opcodes that can be combined with a MUL
2414 static bool isCombineInstrCandidate64(unsigned Opc) {
2416 case AArch64::ADDXrr:
2417 case AArch64::ADDXri:
2418 case AArch64::SUBXrr:
2419 case AArch64::ADDSXrr:
2420 case AArch64::ADDSXri:
2421 case AArch64::SUBSXrr:
2422 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2423 case AArch64::SUBXri:
2424 case AArch64::SUBSXri:
2432 // Opcodes that can be combined with a MUL
2433 static bool isCombineInstrCandidate(unsigned Opc) {
2434 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
2437 static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
2438 unsigned MulOpc, unsigned ZeroReg) {
2439 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2440 MachineInstr *MI = nullptr;
2441 // We need a virtual register definition.
2442 if (MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg()))
2443 MI = MRI.getUniqueVRegDef(MO.getReg());
2444 // And it needs to be in the trace (otherwise, it won't have a depth).
2445 if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != MulOpc)
2448 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
2449 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
2450 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
2452 // The third input reg must be zero.
2453 if (MI->getOperand(3).getReg() != ZeroReg)
2456 // Must only used by the user we combine with.
2457 if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2463 /// hasPattern - return true when there is potentially a faster code sequence
2464 /// for an instruction chain ending in \p Root. All potential patterns are
2466 /// in the \p Pattern vector. Pattern should be sorted in priority order since
2467 /// the pattern evaluator stops checking as soon as it finds a faster sequence.
2469 bool AArch64InstrInfo::hasPattern(
2471 SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Pattern) const {
2472 unsigned Opc = Root.getOpcode();
2473 MachineBasicBlock &MBB = *Root.getParent();
2476 if (!isCombineInstrCandidate(Opc))
2478 if (isCombineInstrSettingFlag(Opc)) {
2479 int Cmp_NZCV = Root.findRegisterDefOperandIdx(AArch64::NZCV, true);
2480 // When NZCV is live bail out.
2483 unsigned NewOpc = convertFlagSettingOpcode(&Root);
2484 // When opcode can't change bail out.
2485 // CHECKME: do we miss any cases for opcode conversion?
2494 case AArch64::ADDWrr:
2495 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
2496 "ADDWrr does not have register operands");
2497 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2499 Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP1);
2502 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
2504 Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP2);
2508 case AArch64::ADDXrr:
2509 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2511 Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP1);
2514 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
2516 Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP2);
2520 case AArch64::SUBWrr:
2521 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2523 Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP1);
2526 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
2528 Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP2);
2532 case AArch64::SUBXrr:
2533 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2535 Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP1);
2538 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
2540 Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP2);
2544 case AArch64::ADDWri:
2545 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2547 Pattern.push_back(MachineCombinerPattern::MC_MULADDWI_OP1);
2551 case AArch64::ADDXri:
2552 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2554 Pattern.push_back(MachineCombinerPattern::MC_MULADDXI_OP1);
2558 case AArch64::SUBWri:
2559 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2561 Pattern.push_back(MachineCombinerPattern::MC_MULSUBWI_OP1);
2565 case AArch64::SUBXri:
2566 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2568 Pattern.push_back(MachineCombinerPattern::MC_MULSUBXI_OP1);
2576 /// genMadd - Generate madd instruction and combine mul and add.
2580 /// ==> MADD R,A,B,C
2581 /// \param Root is the ADD instruction
2582 /// \param [out] InsInstrs is a vector of machine instructions and will
2583 /// contain the generated madd instruction
2584 /// \param IdxMulOpd is index of operand in Root that is the result of
2585 /// the MUL. In the example above IdxMulOpd is 1.
2586 /// \param MaddOpc the opcode fo the madd instruction
2587 static MachineInstr *genMadd(MachineFunction &MF, MachineRegisterInfo &MRI,
2588 const TargetInstrInfo *TII, MachineInstr &Root,
2589 SmallVectorImpl<MachineInstr *> &InsInstrs,
2590 unsigned IdxMulOpd, unsigned MaddOpc,
2591 const TargetRegisterClass *RC) {
2592 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
2594 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
2595 MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
2596 unsigned ResultReg = Root.getOperand(0).getReg();
2597 unsigned SrcReg0 = MUL->getOperand(1).getReg();
2598 bool Src0IsKill = MUL->getOperand(1).isKill();
2599 unsigned SrcReg1 = MUL->getOperand(2).getReg();
2600 bool Src1IsKill = MUL->getOperand(2).isKill();
2601 unsigned SrcReg2 = Root.getOperand(IdxOtherOpd).getReg();
2602 bool Src2IsKill = Root.getOperand(IdxOtherOpd).isKill();
2604 if (TargetRegisterInfo::isVirtualRegister(ResultReg))
2605 MRI.constrainRegClass(ResultReg, RC);
2606 if (TargetRegisterInfo::isVirtualRegister(SrcReg0))
2607 MRI.constrainRegClass(SrcReg0, RC);
2608 if (TargetRegisterInfo::isVirtualRegister(SrcReg1))
2609 MRI.constrainRegClass(SrcReg1, RC);
2610 if (TargetRegisterInfo::isVirtualRegister(SrcReg2))
2611 MRI.constrainRegClass(SrcReg2, RC);
2613 MachineInstrBuilder MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc),
2615 .addReg(SrcReg0, getKillRegState(Src0IsKill))
2616 .addReg(SrcReg1, getKillRegState(Src1IsKill))
2617 .addReg(SrcReg2, getKillRegState(Src2IsKill));
2619 InsInstrs.push_back(MIB);
2623 /// genMaddR - Generate madd instruction and combine mul and add using
2624 /// an extra virtual register
2625 /// Example - an ADD intermediate needs to be stored in a register:
2628 /// ==> ORR V, ZR, Imm
2629 /// ==> MADD R,A,B,V
2630 /// \param Root is the ADD instruction
2631 /// \param [out] InsInstrs is a vector of machine instructions and will
2632 /// contain the generated madd instruction
2633 /// \param IdxMulOpd is index of operand in Root that is the result of
2634 /// the MUL. In the example above IdxMulOpd is 1.
2635 /// \param MaddOpc the opcode fo the madd instruction
2636 /// \param VR is a virtual register that holds the value of an ADD operand
2637 /// (V in the example above).
2638 static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
2639 const TargetInstrInfo *TII, MachineInstr &Root,
2640 SmallVectorImpl<MachineInstr *> &InsInstrs,
2641 unsigned IdxMulOpd, unsigned MaddOpc,
2642 unsigned VR, const TargetRegisterClass *RC) {
2643 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
2645 MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
2646 unsigned ResultReg = Root.getOperand(0).getReg();
2647 unsigned SrcReg0 = MUL->getOperand(1).getReg();
2648 bool Src0IsKill = MUL->getOperand(1).isKill();
2649 unsigned SrcReg1 = MUL->getOperand(2).getReg();
2650 bool Src1IsKill = MUL->getOperand(2).isKill();
2652 if (TargetRegisterInfo::isVirtualRegister(ResultReg))
2653 MRI.constrainRegClass(ResultReg, RC);
2654 if (TargetRegisterInfo::isVirtualRegister(SrcReg0))
2655 MRI.constrainRegClass(SrcReg0, RC);
2656 if (TargetRegisterInfo::isVirtualRegister(SrcReg1))
2657 MRI.constrainRegClass(SrcReg1, RC);
2658 if (TargetRegisterInfo::isVirtualRegister(VR))
2659 MRI.constrainRegClass(VR, RC);
2661 MachineInstrBuilder MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc),
2663 .addReg(SrcReg0, getKillRegState(Src0IsKill))
2664 .addReg(SrcReg1, getKillRegState(Src1IsKill))
2667 InsInstrs.push_back(MIB);
2671 /// genAlternativeCodeSequence - when hasPattern() finds a pattern
2672 /// this function generates the instructions that could replace the
2673 /// original code sequence
2674 void AArch64InstrInfo::genAlternativeCodeSequence(
2675 MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern,
2676 SmallVectorImpl<MachineInstr *> &InsInstrs,
2677 SmallVectorImpl<MachineInstr *> &DelInstrs,
2678 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
2679 MachineBasicBlock &MBB = *Root.getParent();
2680 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2681 MachineFunction &MF = *MBB.getParent();
2682 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
2685 const TargetRegisterClass *RC;
2691 case MachineCombinerPattern::MC_MULADDW_OP1:
2692 case MachineCombinerPattern::MC_MULADDX_OP1:
2696 // --- Create(MADD);
2697 if (Pattern == MachineCombinerPattern::MC_MULADDW_OP1) {
2698 Opc = AArch64::MADDWrrr;
2699 RC = &AArch64::GPR32RegClass;
2701 Opc = AArch64::MADDXrrr;
2702 RC = &AArch64::GPR64RegClass;
2704 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
2706 case MachineCombinerPattern::MC_MULADDW_OP2:
2707 case MachineCombinerPattern::MC_MULADDX_OP2:
2711 // --- Create(MADD);
2712 if (Pattern == MachineCombinerPattern::MC_MULADDW_OP2) {
2713 Opc = AArch64::MADDWrrr;
2714 RC = &AArch64::GPR32RegClass;
2716 Opc = AArch64::MADDXrrr;
2717 RC = &AArch64::GPR64RegClass;
2719 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
2721 case MachineCombinerPattern::MC_MULADDWI_OP1:
2722 case MachineCombinerPattern::MC_MULADDXI_OP1: {
2725 // ==> ORR V, ZR, Imm
2727 // --- Create(MADD);
2728 const TargetRegisterClass *OrrRC;
2729 unsigned BitSize, OrrOpc, ZeroReg;
2730 if (Pattern == MachineCombinerPattern::MC_MULADDWI_OP1) {
2731 OrrOpc = AArch64::ORRWri;
2732 OrrRC = &AArch64::GPR32spRegClass;
2734 ZeroReg = AArch64::WZR;
2735 Opc = AArch64::MADDWrrr;
2736 RC = &AArch64::GPR32RegClass;
2738 OrrOpc = AArch64::ORRXri;
2739 OrrRC = &AArch64::GPR64spRegClass;
2741 ZeroReg = AArch64::XZR;
2742 Opc = AArch64::MADDXrrr;
2743 RC = &AArch64::GPR64RegClass;
2745 unsigned NewVR = MRI.createVirtualRegister(OrrRC);
2746 uint64_t Imm = Root.getOperand(2).getImm();
2748 if (Root.getOperand(3).isImm()) {
2749 unsigned Val = Root.getOperand(3).getImm();
2752 uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
2754 if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
2755 MachineInstrBuilder MIB1 =
2756 BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
2759 InsInstrs.push_back(MIB1);
2760 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2761 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
2765 case MachineCombinerPattern::MC_MULSUBW_OP1:
2766 case MachineCombinerPattern::MC_MULSUBX_OP1: {
2770 // ==> MADD R,A,B,V // = -C + A*B
2771 // --- Create(MADD);
2772 const TargetRegisterClass *SubRC;
2773 unsigned SubOpc, ZeroReg;
2774 if (Pattern == MachineCombinerPattern::MC_MULSUBW_OP1) {
2775 SubOpc = AArch64::SUBWrr;
2776 SubRC = &AArch64::GPR32spRegClass;
2777 ZeroReg = AArch64::WZR;
2778 Opc = AArch64::MADDWrrr;
2779 RC = &AArch64::GPR32RegClass;
2781 SubOpc = AArch64::SUBXrr;
2782 SubRC = &AArch64::GPR64spRegClass;
2783 ZeroReg = AArch64::XZR;
2784 Opc = AArch64::MADDXrrr;
2785 RC = &AArch64::GPR64RegClass;
2787 unsigned NewVR = MRI.createVirtualRegister(SubRC);
2789 MachineInstrBuilder MIB1 =
2790 BuildMI(MF, Root.getDebugLoc(), TII->get(SubOpc), NewVR)
2792 .addOperand(Root.getOperand(2));
2793 InsInstrs.push_back(MIB1);
2794 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2795 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
2798 case MachineCombinerPattern::MC_MULSUBW_OP2:
2799 case MachineCombinerPattern::MC_MULSUBX_OP2:
2802 // ==> MSUB R,A,B,C (computes C - A*B)
2803 // --- Create(MSUB);
2804 if (Pattern == MachineCombinerPattern::MC_MULSUBW_OP2) {
2805 Opc = AArch64::MSUBWrrr;
2806 RC = &AArch64::GPR32RegClass;
2808 Opc = AArch64::MSUBXrrr;
2809 RC = &AArch64::GPR64RegClass;
2811 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
2813 case MachineCombinerPattern::MC_MULSUBWI_OP1:
2814 case MachineCombinerPattern::MC_MULSUBXI_OP1: {
2817 // ==> ORR V, ZR, -Imm
2818 // ==> MADD R,A,B,V // = -Imm + A*B
2819 // --- Create(MADD);
2820 const TargetRegisterClass *OrrRC;
2821 unsigned BitSize, OrrOpc, ZeroReg;
2822 if (Pattern == MachineCombinerPattern::MC_MULSUBWI_OP1) {
2823 OrrOpc = AArch64::ORRWri;
2824 OrrRC = &AArch64::GPR32spRegClass;
2826 ZeroReg = AArch64::WZR;
2827 Opc = AArch64::MADDWrrr;
2828 RC = &AArch64::GPR32RegClass;
2830 OrrOpc = AArch64::ORRXri;
2831 OrrRC = &AArch64::GPR64spRegClass;
2833 ZeroReg = AArch64::XZR;
2834 Opc = AArch64::MADDXrrr;
2835 RC = &AArch64::GPR64RegClass;
2837 unsigned NewVR = MRI.createVirtualRegister(OrrRC);
2838 int Imm = Root.getOperand(2).getImm();
2839 if (Root.getOperand(3).isImm()) {
2840 unsigned Val = Root.getOperand(3).getImm();
2843 uint64_t UImm = -Imm << (64 - BitSize) >> (64 - BitSize);
2845 if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
2846 MachineInstrBuilder MIB1 =
2847 BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
2850 InsInstrs.push_back(MIB1);
2851 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2852 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
2856 } // end switch (Pattern)
2857 // Record MUL and ADD/SUB for deletion
2858 DelInstrs.push_back(MUL);
2859 DelInstrs.push_back(&Root);
2864 /// \brief Replace csincr-branch sequence by simple conditional branch
2868 /// csinc w9, wzr, wzr, <condition code>
2869 /// tbnz w9, #0, 0x44
2871 /// b.<inverted condition code>
2874 /// csinc w9, wzr, wzr, <condition code>
2875 /// tbz w9, #0, 0x44
2877 /// b.<condition code>
2879 /// \param MI Conditional Branch
2880 /// \return True when the simple conditional branch is generated
2882 bool AArch64InstrInfo::optimizeCondBranch(MachineInstr *MI) const {
2883 bool IsNegativeBranch = false;
2884 bool IsTestAndBranch = false;
2885 unsigned TargetBBInMI = 0;
2886 switch (MI->getOpcode()) {
2888 llvm_unreachable("Unknown branch instruction?");
2895 case AArch64::CBNZW:
2896 case AArch64::CBNZX:
2898 IsNegativeBranch = true;
2903 IsTestAndBranch = true;
2905 case AArch64::TBNZW:
2906 case AArch64::TBNZX:
2908 IsNegativeBranch = true;
2909 IsTestAndBranch = true;
2912 // So we increment a zero register and test for bits other
2913 // than bit 0? Conservatively bail out in case the verifier
2914 // missed this case.
2915 if (IsTestAndBranch && MI->getOperand(1).getImm())
2919 assert(MI->getParent() && "Incomplete machine instruciton\n");
2920 MachineBasicBlock *MBB = MI->getParent();
2921 MachineFunction *MF = MBB->getParent();
2922 MachineRegisterInfo *MRI = &MF->getRegInfo();
2923 unsigned VReg = MI->getOperand(0).getReg();
2924 if (!TargetRegisterInfo::isVirtualRegister(VReg))
2927 MachineInstr *DefMI = MRI->getVRegDef(VReg);
2930 if (!(DefMI->getOpcode() == AArch64::CSINCWr &&
2931 DefMI->getOperand(1).getReg() == AArch64::WZR &&
2932 DefMI->getOperand(2).getReg() == AArch64::WZR) &&
2933 !(DefMI->getOpcode() == AArch64::CSINCXr &&
2934 DefMI->getOperand(1).getReg() == AArch64::XZR &&
2935 DefMI->getOperand(2).getReg() == AArch64::XZR))
2938 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) != -1)
2941 AArch64CC::CondCode CC =
2942 (AArch64CC::CondCode)DefMI->getOperand(3).getImm();
2943 bool CheckOnlyCCWrites = true;
2944 // Convert only when the condition code is not modified between
2945 // the CSINC and the branch. The CC may be used by other
2946 // instructions in between.
2947 if (modifiesConditionCode(DefMI, MI, CheckOnlyCCWrites, &getRegisterInfo()))
2949 MachineBasicBlock &RefToMBB = *MBB;
2950 MachineBasicBlock *TBB = MI->getOperand(TargetBBInMI).getMBB();
2951 DebugLoc DL = MI->getDebugLoc();
2952 if (IsNegativeBranch)
2953 CC = AArch64CC::getInvertedCondCode(CC);
2954 BuildMI(RefToMBB, MI, DL, get(AArch64::Bcc)).addImm(CC).addMBB(TBB);
2955 MI->eraseFromParent();