1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the AArch64 implementation of TargetFrameLowering class.
12 // On AArch64, stack frames are structured as follows:
14 // The stack grows downward.
16 // All of the individual frame areas on the frame below are optional, i.e. it's
17 // possible to create a function so that the particular area isn't present
20 // At function entry, the "frame" looks as follows:
23 // |-----------------------------------|
25 // | arguments passed on the stack |
27 // |-----------------------------------| <- sp
31 // After the prologue has run, the frame has the following general structure.
32 // Note that this doesn't depict the case where a red-zone is used. Also,
33 // technically the last frame area (VLAs) doesn't get created until in the
34 // main function body, after the prologue is run. However, it's depicted here
38 // |-----------------------------------|
40 // | arguments passed on the stack |
42 // |-----------------------------------|
44 // | prev_fp, prev_lr |
45 // | (a.k.a. "frame record") |
46 // |-----------------------------------| <- fp(=x29)
48 // | other callee-saved registers |
50 // |-----------------------------------|
51 // |.empty.space.to.make.part.below....|
52 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
53 // |.the.standard.16-byte.alignment....| compile time; if present)
54 // |-----------------------------------|
56 // | local variables of fixed size |
57 // | including spill slots |
58 // |-----------------------------------| <- bp(not defined by ABI,
59 // |.variable-sized.local.variables....| LLVM chooses X19)
60 // |.(VLAs)............................| (size of this area is unknown at
61 // |...................................| compile time)
62 // |-----------------------------------| <- sp
66 // To access the data in a frame, at-compile time, a constant offset must be
67 // computable from one of the pointers (fp, bp, sp) to access it. The size
68 // of the areas with a dotted background cannot be computed at compile-time
69 // if they are present, making it required to have all three of fp, bp and
70 // sp to be set up to be able to access all contents in the frame areas,
71 // assuming all of the frame areas are non-empty.
73 // For most functions, some of the frame areas are empty. For those functions,
74 // it may not be necessary to set up fp or bp:
75 // * A base pointer is definitly needed when there are both VLAs and local
76 // variables with more-than-default alignment requirements.
77 // * A frame pointer is definitly needed when there are local variables with
78 // more-than-default alignment requirements.
80 // In some cases when a base pointer is not strictly needed, it is generated
81 // anyway when offsets from the frame pointer to access local variables become
82 // so large that the offset can't be encoded in the immediate fields of loads
85 // FIXME: also explain the redzone concept.
86 // FIXME: also explain the concept of reserved call frames.
88 //===----------------------------------------------------------------------===//
90 #include "AArch64FrameLowering.h"
91 #include "AArch64InstrInfo.h"
92 #include "AArch64MachineFunctionInfo.h"
93 #include "AArch64Subtarget.h"
94 #include "AArch64TargetMachine.h"
95 #include "llvm/ADT/Statistic.h"
96 #include "llvm/CodeGen/MachineFrameInfo.h"
97 #include "llvm/CodeGen/MachineFunction.h"
98 #include "llvm/CodeGen/MachineInstrBuilder.h"
99 #include "llvm/CodeGen/MachineModuleInfo.h"
100 #include "llvm/CodeGen/MachineRegisterInfo.h"
101 #include "llvm/CodeGen/RegisterScavenging.h"
102 #include "llvm/IR/DataLayout.h"
103 #include "llvm/IR/Function.h"
104 #include "llvm/Support/CommandLine.h"
105 #include "llvm/Support/Debug.h"
106 #include "llvm/Support/raw_ostream.h"
108 using namespace llvm;
110 #define DEBUG_TYPE "frame-info"
112 static cl::opt<bool> EnableRedZone("aarch64-redzone",
113 cl::desc("enable use of redzone on AArch64"),
114 cl::init(false), cl::Hidden);
116 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
118 bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
121 // Don't use the red zone if the function explicitly asks us not to.
122 // This is typically used for kernel code.
123 if (MF.getFunction()->hasFnAttribute(Attribute::NoRedZone))
126 const MachineFrameInfo *MFI = MF.getFrameInfo();
127 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
128 unsigned NumBytes = AFI->getLocalStackSize();
130 // Note: currently hasFP() is always true for hasCalls(), but that's an
131 // implementation detail of the current code, not a strict requirement,
132 // so stay safe here and check both.
133 if (MFI->hasCalls() || hasFP(MF) || NumBytes > 128)
138 /// hasFP - Return true if the specified function should have a dedicated frame
139 /// pointer register.
140 bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
141 const MachineFrameInfo *MFI = MF.getFrameInfo();
142 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
143 return (MFI->hasCalls() || MFI->hasVarSizedObjects() ||
144 MFI->isFrameAddressTaken() || MFI->hasStackMap() ||
145 MFI->hasPatchPoint() || RegInfo->needsStackRealignment(MF));
148 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
149 /// not required, we reserve argument space for call sites in the function
150 /// immediately on entry to the current function. This eliminates the need for
151 /// add/sub sp brackets around call sites. Returns true if the call frame is
152 /// included as part of the stack frame.
154 AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
155 return !MF.getFrameInfo()->hasVarSizedObjects();
158 void AArch64FrameLowering::eliminateCallFramePseudoInstr(
159 MachineFunction &MF, MachineBasicBlock &MBB,
160 MachineBasicBlock::iterator I) const {
161 const AArch64InstrInfo *TII =
162 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
163 DebugLoc DL = I->getDebugLoc();
164 unsigned Opc = I->getOpcode();
165 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
166 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
168 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
169 if (!TFI->hasReservedCallFrame(MF)) {
170 unsigned Align = getStackAlignment();
172 int64_t Amount = I->getOperand(0).getImm();
173 Amount = RoundUpToAlignment(Amount, Align);
177 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
178 // doesn't have to pop anything), then the first operand will be zero too so
179 // this adjustment is a no-op.
180 if (CalleePopAmount == 0) {
181 // FIXME: in-function stack adjustment for calls is limited to 24-bits
182 // because there's no guaranteed temporary register available.
184 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
185 // 1) For offset <= 12-bit, we use LSL #0
186 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
187 // LSL #0, and the other uses LSL #12.
189 // Mostly call frames will be allocated at the start of a function so
190 // this is OK, but it is a limitation that needs dealing with.
191 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
192 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
194 } else if (CalleePopAmount != 0) {
195 // If the calling convention demands that the callee pops arguments from the
196 // stack, we want to add it back if we have a reserved call frame.
197 assert(CalleePopAmount < 0xffffff && "call frame too large");
198 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
204 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
205 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
206 unsigned FramePtr) const {
207 MachineFunction &MF = *MBB.getParent();
208 MachineFrameInfo *MFI = MF.getFrameInfo();
209 MachineModuleInfo &MMI = MF.getMMI();
210 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
211 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
212 DebugLoc DL = MBB.findDebugLoc(MBBI);
214 // Add callee saved registers to move list.
215 const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
219 const DataLayout *TD = MF.getTarget().getDataLayout();
220 bool HasFP = hasFP(MF);
222 // Calculate amount of bytes used for return address storing.
223 int stackGrowth = -TD->getPointerSize(0);
225 // Calculate offsets.
226 int64_t saveAreaOffset = (HasFP ? 2 : 1) * stackGrowth;
227 unsigned TotalSkipped = 0;
228 for (const auto &Info : CSI) {
229 unsigned Reg = Info.getReg();
230 int64_t Offset = MFI->getObjectOffset(Info.getFrameIdx()) -
231 getOffsetOfLocalArea() + saveAreaOffset;
233 // Don't output a new CFI directive if we're re-saving the frame pointer or
234 // link register. This happens when the PrologEpilogInserter has inserted an
235 // extra "STP" of the frame pointer and link register -- the "emitPrologue"
236 // method automatically generates the directives when frame pointers are
237 // used. If we generate CFI directives for the extra "STP"s, the linker will
238 // lose track of the correct values for the frame pointer and link register.
239 if (HasFP && (FramePtr == Reg || Reg == AArch64::LR)) {
240 TotalSkipped += stackGrowth;
244 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
245 unsigned CFIIndex = MMI.addFrameInst(MCCFIInstruction::createOffset(
246 nullptr, DwarfReg, Offset - TotalSkipped));
247 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
248 .addCFIIndex(CFIIndex)
249 .setMIFlags(MachineInstr::FrameSetup);
253 /// Get FPOffset by analyzing the first instruction.
254 static int getFPOffsetInPrologue(MachineInstr *MBBI) {
255 // First instruction must a) allocate the stack and b) have an immediate
256 // that is a multiple of -2.
257 assert(((MBBI->getOpcode() == AArch64::STPXpre ||
258 MBBI->getOpcode() == AArch64::STPDpre) &&
259 MBBI->getOperand(3).getReg() == AArch64::SP &&
260 MBBI->getOperand(4).getImm() < 0 &&
261 (MBBI->getOperand(4).getImm() & 1) == 0));
263 // Frame pointer is fp = sp - 16. Since the STPXpre subtracts the space
264 // required for the callee saved register area we get the frame pointer
265 // by addding that offset - 16 = -getImm()*8 - 2*8 = -(getImm() + 2) * 8.
266 int FPOffset = -(MBBI->getOperand(4).getImm() + 2) * 8;
267 assert(FPOffset >= 0 && "Bad Framepointer Offset");
271 static bool isCSSave(MachineInstr *MBBI) {
272 return MBBI->getOpcode() == AArch64::STPXi ||
273 MBBI->getOpcode() == AArch64::STPDi ||
274 MBBI->getOpcode() == AArch64::STPXpre ||
275 MBBI->getOpcode() == AArch64::STPDpre;
278 void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
279 MachineBasicBlock &MBB) const {
280 MachineBasicBlock::iterator MBBI = MBB.begin();
281 const MachineFrameInfo *MFI = MF.getFrameInfo();
282 const Function *Fn = MF.getFunction();
283 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
284 MF.getSubtarget().getRegisterInfo());
285 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
286 MachineModuleInfo &MMI = MF.getMMI();
287 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
288 bool needsFrameMoves = MMI.hasDebugInfo() || Fn->needsUnwindTableEntry();
289 bool HasFP = hasFP(MF);
290 DebugLoc DL = MBB.findDebugLoc(MBBI);
292 // All calls are tail calls in GHC calling conv, and functions have no
293 // prologue/epilogue.
294 if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
297 int NumBytes = (int)MFI->getStackSize();
298 if (!AFI->hasStackFrame()) {
299 assert(!HasFP && "unexpected function without stack frame but with FP");
301 // All of the stack allocation is for locals.
302 AFI->setLocalStackSize(NumBytes);
304 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
305 MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
307 // REDZONE: If the stack size is less than 128 bytes, we don't need
308 // to actually allocate.
309 if (NumBytes && !canUseRedZone(MF)) {
310 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
311 MachineInstr::FrameSetup);
313 // Encode the stack size of the leaf function.
314 unsigned CFIIndex = MMI.addFrameInst(
315 MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
316 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
317 .addCFIIndex(CFIIndex)
318 .setMIFlags(MachineInstr::FrameSetup);
319 } else if (NumBytes) {
320 ++NumRedZoneFunctions;
326 // Only set up FP if we actually need to.
329 FPOffset = getFPOffsetInPrologue(MBBI);
331 // Move past the saves of the callee-saved registers.
332 while (isCSSave(MBBI)) {
336 assert(NumBytes >= 0 && "Negative stack allocation size!?");
338 // Issue sub fp, sp, FPOffset or
339 // mov fp,sp when FPOffset is zero.
340 // Note: All stores of callee-saved registers are marked as "FrameSetup".
341 // This code marks the instruction(s) that set the FP also.
342 emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
343 MachineInstr::FrameSetup);
346 // All of the remaining stack allocations are for locals.
347 AFI->setLocalStackSize(NumBytes);
349 // Allocate space for the rest of the frame.
351 const unsigned Alignment = MFI->getMaxAlignment();
352 const bool NeedsRealignment = (Alignment > 16);
353 unsigned scratchSPReg = AArch64::SP;
354 if (NeedsRealignment) {
355 // Use the first callee-saved register as a scratch register
356 assert(MF.getRegInfo().isPhysRegUsed(AArch64::X9) &&
357 "No scratch register to align SP!");
358 scratchSPReg = AArch64::X9;
361 // If we're a leaf function, try using the red zone.
362 if (NumBytes && !canUseRedZone(MF))
363 // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
364 // the correct value here, as NumBytes also includes padding bytes,
365 // which shouldn't be counted here.
366 emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
367 MachineInstr::FrameSetup);
369 assert(!(NeedsRealignment && NumBytes==0) &&
370 "NumBytes should never be 0 when realignment is needed");
372 if (NumBytes && NeedsRealignment) {
373 const unsigned NrBitsToZero = countTrailingZeros(Alignment);
374 assert(NrBitsToZero > 1);
375 assert(scratchSPReg != AArch64::SP);
377 // SUB X9, SP, NumBytes
378 // -- X9 is temporary register, so shouldn't contain any live data here,
379 // -- free to use. This is already produced by emitFrameOffset above.
380 // AND SP, X9, 0b11111...0000
381 // The logical immediates have a non-trivial encoding. The following
382 // formula computes the encoded immediate with all ones but
383 // NrBitsToZero zero bits as least significant bits.
384 uint32_t andMaskEncoded =
386 | ((64-NrBitsToZero) << 6) // immr
387 | ((64-NrBitsToZero-1) << 0) // imms
389 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
390 .addReg(scratchSPReg, RegState::Kill)
391 .addImm(andMaskEncoded);
394 // If we need a base pointer, set it up here. It's whatever the value of the
395 // stack pointer is at this point. Any variable size objects will be allocated
396 // after this, so we can still use the base pointer to reference locals.
398 // FIXME: Clarify FrameSetup flags here.
399 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
401 if (RegInfo->hasBasePointer(MF)) {
402 TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
406 if (needsFrameMoves) {
407 const DataLayout *TD = MF.getTarget().getDataLayout();
408 const int StackGrowth = -TD->getPointerSize(0);
409 unsigned FramePtr = RegInfo->getFrameRegister(MF);
410 // An example of the prologue:
417 // .cfi_personality 155, ___gxx_personality_v0
419 // .cfi_lsda 16, Lexception33
421 // stp xa,bx, [sp, -#offset]!
423 // stp x28, x27, [sp, #offset-32]
424 // stp fp, lr, [sp, #offset-16]
425 // add fp, sp, #offset - 16
429 // +-------------------------------------------+
430 // 10000 | ........ | ........ | ........ | ........ |
431 // 10004 | ........ | ........ | ........ | ........ |
432 // +-------------------------------------------+
433 // 10008 | ........ | ........ | ........ | ........ |
434 // 1000c | ........ | ........ | ........ | ........ |
435 // +===========================================+
436 // 10010 | X28 Register |
437 // 10014 | X28 Register |
438 // +-------------------------------------------+
439 // 10018 | X27 Register |
440 // 1001c | X27 Register |
441 // +===========================================+
442 // 10020 | Frame Pointer |
443 // 10024 | Frame Pointer |
444 // +-------------------------------------------+
445 // 10028 | Link Register |
446 // 1002c | Link Register |
447 // +===========================================+
448 // 10030 | ........ | ........ | ........ | ........ |
449 // 10034 | ........ | ........ | ........ | ........ |
450 // +-------------------------------------------+
451 // 10038 | ........ | ........ | ........ | ........ |
452 // 1003c | ........ | ........ | ........ | ........ |
453 // +-------------------------------------------+
455 // [sp] = 10030 :: >>initial value<<
456 // sp = 10020 :: stp fp, lr, [sp, #-16]!
457 // fp = sp == 10020 :: mov fp, sp
458 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
459 // sp == 10010 :: >>final value<<
461 // The frame pointer (w29) points to address 10020. If we use an offset of
462 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
463 // for w27, and -32 for w28:
466 // .cfi_def_cfa w29, 16
468 // .cfi_offset w30, -8
470 // .cfi_offset w29, -16
472 // .cfi_offset w27, -24
474 // .cfi_offset w28, -32
477 // Define the current CFA rule to use the provided FP.
478 unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
479 unsigned CFIIndex = MMI.addFrameInst(
480 MCCFIInstruction::createDefCfa(nullptr, Reg, 2 * StackGrowth));
481 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
482 .addCFIIndex(CFIIndex)
483 .setMIFlags(MachineInstr::FrameSetup);
485 // Record the location of the stored LR
486 unsigned LR = RegInfo->getDwarfRegNum(AArch64::LR, true);
487 CFIIndex = MMI.addFrameInst(
488 MCCFIInstruction::createOffset(nullptr, LR, StackGrowth));
489 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
490 .addCFIIndex(CFIIndex)
491 .setMIFlags(MachineInstr::FrameSetup);
493 // Record the location of the stored FP
494 CFIIndex = MMI.addFrameInst(
495 MCCFIInstruction::createOffset(nullptr, Reg, 2 * StackGrowth));
496 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
497 .addCFIIndex(CFIIndex)
498 .setMIFlags(MachineInstr::FrameSetup);
500 // Encode the stack size of the leaf function.
501 unsigned CFIIndex = MMI.addFrameInst(
502 MCCFIInstruction::createDefCfaOffset(nullptr, -MFI->getStackSize()));
503 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
504 .addCFIIndex(CFIIndex)
505 .setMIFlags(MachineInstr::FrameSetup);
508 // Now emit the moves for whatever callee saved regs we have.
509 emitCalleeSavedFrameMoves(MBB, MBBI, FramePtr);
513 static bool isCalleeSavedRegister(unsigned Reg, const MCPhysReg *CSRegs) {
514 for (unsigned i = 0; CSRegs[i]; ++i)
515 if (Reg == CSRegs[i])
520 static bool isCSRestore(MachineInstr *MI, const MCPhysReg *CSRegs) {
522 if (MI->getOpcode() == AArch64::LDPXpost ||
523 MI->getOpcode() == AArch64::LDPDpost)
526 if (MI->getOpcode() == AArch64::LDPXpost ||
527 MI->getOpcode() == AArch64::LDPDpost ||
528 MI->getOpcode() == AArch64::LDPXi || MI->getOpcode() == AArch64::LDPDi) {
529 if (!isCalleeSavedRegister(MI->getOperand(RtIdx).getReg(), CSRegs) ||
530 !isCalleeSavedRegister(MI->getOperand(RtIdx + 1).getReg(), CSRegs) ||
531 MI->getOperand(RtIdx + 2).getReg() != AArch64::SP)
539 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
540 MachineBasicBlock &MBB) const {
541 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
542 MachineFrameInfo *MFI = MF.getFrameInfo();
543 const AArch64InstrInfo *TII =
544 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
545 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
546 MF.getSubtarget().getRegisterInfo());
548 bool IsTailCallReturn = false;
549 if (MBB.end() != MBBI) {
550 DL = MBBI->getDebugLoc();
551 unsigned RetOpcode = MBBI->getOpcode();
552 IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
553 RetOpcode == AArch64::TCRETURNri;
555 int NumBytes = MFI->getStackSize();
556 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
558 // All calls are tail calls in GHC calling conv, and functions have no
559 // prologue/epilogue.
560 if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
563 // Initial and residual are named for consistency with the prologue. Note that
564 // in the epilogue, the residual adjustment is executed first.
565 uint64_t ArgumentPopSize = 0;
566 if (IsTailCallReturn) {
567 MachineOperand &StackAdjust = MBBI->getOperand(1);
569 // For a tail-call in a callee-pops-arguments environment, some or all of
570 // the stack may actually be in use for the call's arguments, this is
571 // calculated during LowerCall and consumed here...
572 ArgumentPopSize = StackAdjust.getImm();
574 // ... otherwise the amount to pop is *all* of the argument space,
575 // conveniently stored in the MachineFunctionInfo by
576 // LowerFormalArguments. This will, of course, be zero for the C calling
578 ArgumentPopSize = AFI->getArgumentStackToRestore();
581 // The stack frame should be like below,
583 // ---------------------- ---
585 // | BytesInStackArgArea| CalleeArgStackSize
586 // | (NumReusableBytes) | (of tail call)
589 // ---------------------| --- |
591 // | CalleeSavedReg | | |
592 // | (NumRestores * 16) | | |
594 // ---------------------| | NumBytes
595 // | | StackSize (StackAdjustUp)
596 // | LocalStackSize | | |
597 // | (covering callee | | |
600 // ---------------------- --- ---
602 // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
603 // = StackSize + ArgumentPopSize
605 // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
606 // it as the 2nd argument of AArch64ISD::TC_RETURN.
607 NumBytes += ArgumentPopSize;
609 unsigned NumRestores = 0;
610 // Move past the restores of the callee-saved registers.
611 MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
612 const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
613 if (LastPopI != MBB.begin()) {
617 } while (LastPopI != MBB.begin() && isCSRestore(LastPopI, CSRegs));
618 if (!isCSRestore(LastPopI, CSRegs)) {
623 NumBytes -= NumRestores * 16;
624 assert(NumBytes >= 0 && "Negative stack allocation size!?");
627 // If this was a redzone leaf function, we don't need to restore the
629 if (!canUseRedZone(MF))
630 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes,
635 // Restore the original stack pointer.
636 // FIXME: Rather than doing the math here, we should instead just use
637 // non-post-indexed loads for the restores if we aren't actually going to
638 // be able to save any instructions.
639 if (NumBytes || MFI->hasVarSizedObjects())
640 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
641 -(NumRestores - 1) * 16, TII, MachineInstr::NoFlags);
644 /// getFrameIndexOffset - Returns the displacement from the frame register to
645 /// the stack frame of the specified index.
646 int AArch64FrameLowering::getFrameIndexOffset(const MachineFunction &MF,
649 return getFrameIndexReference(MF, FI, FrameReg);
652 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
653 /// debug info. It's the same as what we use for resolving the code-gen
654 /// references for now. FIXME: This can go wrong when references are
655 /// SP-relative and simple call frames aren't used.
656 int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
658 unsigned &FrameReg) const {
659 return resolveFrameIndexReference(MF, FI, FrameReg);
662 int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
663 int FI, unsigned &FrameReg,
664 bool PreferFP) const {
665 const MachineFrameInfo *MFI = MF.getFrameInfo();
666 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
667 MF.getSubtarget().getRegisterInfo());
668 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
669 int FPOffset = MFI->getObjectOffset(FI) + 16;
670 int Offset = MFI->getObjectOffset(FI) + MFI->getStackSize();
671 bool isFixed = MFI->isFixedObjectIndex(FI);
673 // Use frame pointer to reference fixed objects. Use it for locals if
674 // there are VLAs or a dynamically realigned SP (and thus the SP isn't
675 // reliable as a base). Make sure useFPForScavengingIndex() does the
676 // right thing for the emergency spill slot.
678 if (AFI->hasStackFrame()) {
679 // Note: Keeping the following as multiple 'if' statements rather than
680 // merging to a single expression for readability.
682 // Argument access should always use the FP.
685 } else if (hasFP(MF) && !RegInfo->hasBasePointer(MF) &&
686 !RegInfo->needsStackRealignment(MF)) {
687 // Use SP or FP, whichever gives us the best chance of the offset
688 // being in range for direct access. If the FPOffset is positive,
689 // that'll always be best, as the SP will be even further away.
690 // If the FPOffset is negative, we have to keep in mind that the
691 // available offset range for negative offsets is smaller than for
692 // positive ones. If we have variable sized objects, we're stuck with
693 // using the FP regardless, though, as the SP offset is unknown
694 // and we don't have a base pointer available. If an offset is
695 // available via the FP and the SP, use whichever is closest.
696 if (PreferFP || MFI->hasVarSizedObjects() || FPOffset >= 0 ||
697 (FPOffset >= -256 && Offset > -FPOffset))
702 assert((isFixed || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
703 "In the presence of dynamic stack pointer realignment, "
704 "non-argument objects cannot be accessed through the frame pointer");
707 FrameReg = RegInfo->getFrameRegister(MF);
711 // Use the base pointer if we have one.
712 if (RegInfo->hasBasePointer(MF))
713 FrameReg = RegInfo->getBaseRegister();
715 FrameReg = AArch64::SP;
716 // If we're using the red zone for this function, the SP won't actually
717 // be adjusted, so the offsets will be negative. They're also all
718 // within range of the signed 9-bit immediate instructions.
719 if (canUseRedZone(MF))
720 Offset -= AFI->getLocalStackSize();
726 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
727 if (Reg != AArch64::LR)
728 return getKillRegState(true);
730 // LR maybe referred to later by an @llvm.returnaddress intrinsic.
731 bool LRLiveIn = MF.getRegInfo().isLiveIn(AArch64::LR);
732 bool LRKill = !(LRLiveIn && MF.getFrameInfo()->isReturnAddressTaken());
733 return getKillRegState(LRKill);
736 bool AArch64FrameLowering::spillCalleeSavedRegisters(
737 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
738 const std::vector<CalleeSavedInfo> &CSI,
739 const TargetRegisterInfo *TRI) const {
740 MachineFunction &MF = *MBB.getParent();
741 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
742 unsigned Count = CSI.size();
744 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
747 DL = MI->getDebugLoc();
749 for (unsigned i = 0; i < Count; i += 2) {
750 unsigned idx = Count - i - 2;
751 unsigned Reg1 = CSI[idx].getReg();
752 unsigned Reg2 = CSI[idx + 1].getReg();
753 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
754 // list to come in sorted by frame index so that we can issue the store
755 // pair instructions directly. Assert if we see anything otherwise.
757 // The order of the registers in the list is controlled by
758 // getCalleeSavedRegs(), so they will always be in-order, as well.
759 assert(CSI[idx].getFrameIdx() + 1 == CSI[idx + 1].getFrameIdx() &&
760 "Out of order callee saved regs!");
762 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
763 assert((i & 1) == 0 && "Odd index for callee-saved reg spill!");
764 // Issue sequence of non-sp increment and pi sp spills for cs regs. The
765 // first spill is a pre-increment that allocates the stack.
767 // stp x22, x21, [sp, #-48]! // addImm(-6)
768 // stp x20, x19, [sp, #16] // addImm(+2)
769 // stp fp, lr, [sp, #32] // addImm(+4)
770 // Rationale: This sequence saves uop updates compared to a sequence of
771 // pre-increment spills like stp xi,xj,[sp,#-16]!
772 // Note: Similar rational and sequence for restores in epilog.
773 if (AArch64::GPR64RegClass.contains(Reg1)) {
774 assert(AArch64::GPR64RegClass.contains(Reg2) &&
775 "Expected GPR64 callee-saved register pair!");
776 // For first spill use pre-increment store.
778 StrOpc = AArch64::STPXpre;
780 StrOpc = AArch64::STPXi;
781 } else if (AArch64::FPR64RegClass.contains(Reg1)) {
782 assert(AArch64::FPR64RegClass.contains(Reg2) &&
783 "Expected FPR64 callee-saved register pair!");
784 // For first spill use pre-increment store.
786 StrOpc = AArch64::STPDpre;
788 StrOpc = AArch64::STPDi;
790 llvm_unreachable("Unexpected callee saved register!");
791 DEBUG(dbgs() << "CSR spill: (" << TRI->getName(Reg1) << ", "
792 << TRI->getName(Reg2) << ") -> fi#(" << CSI[idx].getFrameIdx()
793 << ", " << CSI[idx + 1].getFrameIdx() << ")\n");
794 // Compute offset: i = 0 => offset = -Count;
795 // i = 2 => offset = -(Count - 2) + Count = 2 = i; etc.
796 const int Offset = (i == 0) ? -Count : i;
797 assert((Offset >= -64 && Offset <= 63) &&
798 "Offset out of bounds for STP immediate");
799 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
800 if (StrOpc == AArch64::STPDpre || StrOpc == AArch64::STPXpre)
801 MIB.addReg(AArch64::SP, RegState::Define);
805 MIB.addReg(Reg2, getPrologueDeath(MF, Reg2))
806 .addReg(Reg1, getPrologueDeath(MF, Reg1))
808 .addImm(Offset) // [sp, #offset * 8], where factor * 8 is implicit
809 .setMIFlag(MachineInstr::FrameSetup);
814 bool AArch64FrameLowering::restoreCalleeSavedRegisters(
815 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
816 const std::vector<CalleeSavedInfo> &CSI,
817 const TargetRegisterInfo *TRI) const {
818 MachineFunction &MF = *MBB.getParent();
819 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
820 unsigned Count = CSI.size();
822 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
825 DL = MI->getDebugLoc();
827 for (unsigned i = 0; i < Count; i += 2) {
828 unsigned Reg1 = CSI[i].getReg();
829 unsigned Reg2 = CSI[i + 1].getReg();
830 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
831 // list to come in sorted by frame index so that we can issue the store
832 // pair instructions directly. Assert if we see anything otherwise.
833 assert(CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx() &&
834 "Out of order callee saved regs!");
835 // Issue sequence of non-sp increment and sp-pi restores for cs regs. Only
836 // the last load is sp-pi post-increment and de-allocates the stack:
838 // ldp fp, lr, [sp, #32] // addImm(+4)
839 // ldp x20, x19, [sp, #16] // addImm(+2)
840 // ldp x22, x21, [sp], #48 // addImm(+6)
841 // Note: see comment in spillCalleeSavedRegisters()
844 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
845 assert((i & 1) == 0 && "Odd index for callee-saved reg spill!");
846 if (AArch64::GPR64RegClass.contains(Reg1)) {
847 assert(AArch64::GPR64RegClass.contains(Reg2) &&
848 "Expected GPR64 callee-saved register pair!");
850 LdrOpc = AArch64::LDPXpost;
852 LdrOpc = AArch64::LDPXi;
853 } else if (AArch64::FPR64RegClass.contains(Reg1)) {
854 assert(AArch64::FPR64RegClass.contains(Reg2) &&
855 "Expected FPR64 callee-saved register pair!");
857 LdrOpc = AArch64::LDPDpost;
859 LdrOpc = AArch64::LDPDi;
861 llvm_unreachable("Unexpected callee saved register!");
862 DEBUG(dbgs() << "CSR restore: (" << TRI->getName(Reg1) << ", "
863 << TRI->getName(Reg2) << ") -> fi#(" << CSI[i].getFrameIdx()
864 << ", " << CSI[i + 1].getFrameIdx() << ")\n");
866 // Compute offset: i = 0 => offset = Count - 2; i = 2 => offset = Count - 4;
868 const int Offset = (i == Count - 2) ? Count : Count - i - 2;
869 assert((Offset >= -64 && Offset <= 63) &&
870 "Offset out of bounds for LDP immediate");
871 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
872 if (LdrOpc == AArch64::LDPXpost || LdrOpc == AArch64::LDPDpost)
873 MIB.addReg(AArch64::SP, RegState::Define);
875 MIB.addReg(Reg2, getDefRegState(true))
876 .addReg(Reg1, getDefRegState(true))
878 .addImm(Offset); // [sp], #offset * 8 or [sp, #offset * 8]
879 // where the factor * 8 is implicit
884 void AArch64FrameLowering::processFunctionBeforeCalleeSavedScan(
885 MachineFunction &MF, RegScavenger *RS) const {
886 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
887 MF.getSubtarget().getRegisterInfo());
888 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
889 MachineRegisterInfo *MRI = &MF.getRegInfo();
890 SmallVector<unsigned, 4> UnspilledCSGPRs;
891 SmallVector<unsigned, 4> UnspilledCSFPRs;
893 // The frame record needs to be created by saving the appropriate registers
895 MRI->setPhysRegUsed(AArch64::FP);
896 MRI->setPhysRegUsed(AArch64::LR);
899 // Spill the BasePtr if it's used. Do this first thing so that the
900 // getCalleeSavedRegs() below will get the right answer.
901 if (RegInfo->hasBasePointer(MF))
902 MRI->setPhysRegUsed(RegInfo->getBaseRegister());
904 if (RegInfo->needsStackRealignment(MF) && !RegInfo->hasBasePointer(MF))
905 MRI->setPhysRegUsed(AArch64::X9);
907 // If any callee-saved registers are used, the frame cannot be eliminated.
908 unsigned NumGPRSpilled = 0;
909 unsigned NumFPRSpilled = 0;
910 bool ExtraCSSpill = false;
911 bool CanEliminateFrame = true;
912 DEBUG(dbgs() << "*** processFunctionBeforeCalleeSavedScan\nUsed CSRs:");
913 const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
915 // Check pairs of consecutive callee-saved registers.
916 for (unsigned i = 0; CSRegs[i]; i += 2) {
917 assert(CSRegs[i + 1] && "Odd number of callee-saved registers!");
919 const unsigned OddReg = CSRegs[i];
920 const unsigned EvenReg = CSRegs[i + 1];
921 assert((AArch64::GPR64RegClass.contains(OddReg) &&
922 AArch64::GPR64RegClass.contains(EvenReg)) ^
923 (AArch64::FPR64RegClass.contains(OddReg) &&
924 AArch64::FPR64RegClass.contains(EvenReg)) &&
925 "Register class mismatch!");
927 const bool OddRegUsed = MRI->isPhysRegUsed(OddReg);
928 const bool EvenRegUsed = MRI->isPhysRegUsed(EvenReg);
930 // Early exit if none of the registers in the register pair is actually
932 if (!OddRegUsed && !EvenRegUsed) {
933 if (AArch64::GPR64RegClass.contains(OddReg)) {
934 UnspilledCSGPRs.push_back(OddReg);
935 UnspilledCSGPRs.push_back(EvenReg);
937 UnspilledCSFPRs.push_back(OddReg);
938 UnspilledCSFPRs.push_back(EvenReg);
943 unsigned Reg = AArch64::NoRegister;
944 // If only one of the registers of the register pair is used, make sure to
945 // mark the other one as used as well.
946 if (OddRegUsed ^ EvenRegUsed) {
947 // Find out which register is the additional spill.
948 Reg = OddRegUsed ? EvenReg : OddReg;
949 MRI->setPhysRegUsed(Reg);
952 DEBUG(dbgs() << ' ' << PrintReg(OddReg, RegInfo));
953 DEBUG(dbgs() << ' ' << PrintReg(EvenReg, RegInfo));
955 assert(((OddReg == AArch64::LR && EvenReg == AArch64::FP) ||
956 (RegInfo->getEncodingValue(OddReg) + 1 ==
957 RegInfo->getEncodingValue(EvenReg))) &&
958 "Register pair of non-adjacent registers!");
959 if (AArch64::GPR64RegClass.contains(OddReg)) {
961 // If it's not a reserved register, we can use it in lieu of an
962 // emergency spill slot for the register scavenger.
963 // FIXME: It would be better to instead keep looking and choose another
964 // unspilled register that isn't reserved, if there is one.
965 if (Reg != AArch64::NoRegister && !RegInfo->isReservedReg(MF, Reg))
970 CanEliminateFrame = false;
973 // FIXME: Set BigStack if any stack slot references may be out of range.
974 // For now, just conservatively guestimate based on unscaled indexing
975 // range. We'll end up allocating an unnecessary spill slot a lot, but
976 // realistically that's not a big deal at this stage of the game.
977 // The CSR spill slots have not been allocated yet, so estimateStackSize
978 // won't include them.
979 MachineFrameInfo *MFI = MF.getFrameInfo();
981 MFI->estimateStackSize(MF) + 8 * (NumGPRSpilled + NumFPRSpilled);
982 DEBUG(dbgs() << "Estimated stack frame size: " << CFSize << " bytes.\n");
983 bool BigStack = (CFSize >= 256);
984 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
985 AFI->setHasStackFrame(true);
987 // Estimate if we might need to scavenge a register at some point in order
988 // to materialize a stack offset. If so, either spill one additional
989 // callee-saved register or reserve a special spill slot to facilitate
990 // register scavenging. If we already spilled an extra callee-saved register
991 // above to keep the number of spills even, we don't need to do anything else
993 if (BigStack && !ExtraCSSpill) {
995 // If we're adding a register to spill here, we have to add two of them
996 // to keep the number of regs to spill even.
997 assert(((UnspilledCSGPRs.size() & 1) == 0) && "Odd number of registers!");
999 while (!UnspilledCSGPRs.empty() && Count < 2) {
1000 unsigned Reg = UnspilledCSGPRs.back();
1001 UnspilledCSGPRs.pop_back();
1002 DEBUG(dbgs() << "Spilling " << PrintReg(Reg, RegInfo)
1003 << " to get a scratch register.\n");
1004 MRI->setPhysRegUsed(Reg);
1005 ExtraCSSpill = true;
1009 // If we didn't find an extra callee-saved register to spill, create
1010 // an emergency spill slot.
1011 if (!ExtraCSSpill) {
1012 const TargetRegisterClass *RC = &AArch64::GPR64RegClass;
1013 int FI = MFI->CreateStackObject(RC->getSize(), RC->getAlignment(), false);
1014 RS->addScavengingFrameIndex(FI);
1015 DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
1016 << " as the emergency spill slot.\n");