1 //===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the X86 implementation of TargetFrameLowering class.
12 //===----------------------------------------------------------------------===//
14 #include "X86FrameLowering.h"
15 #include "X86InstrBuilder.h"
16 #include "X86InstrInfo.h"
17 #include "X86MachineFunctionInfo.h"
18 #include "X86Subtarget.h"
19 #include "X86TargetMachine.h"
20 #include "llvm/ADT/SmallSet.h"
21 #include "llvm/Analysis/LibCallSemantics.h"
22 #include "llvm/CodeGen/MachineFrameInfo.h"
23 #include "llvm/CodeGen/MachineFunction.h"
24 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/MachineModuleInfo.h"
26 #include "llvm/CodeGen/MachineRegisterInfo.h"
27 #include "llvm/CodeGen/WinEHFuncInfo.h"
28 #include "llvm/IR/DataLayout.h"
29 #include "llvm/IR/Function.h"
30 #include "llvm/MC/MCAsmInfo.h"
31 #include "llvm/MC/MCSymbol.h"
32 #include "llvm/Target/TargetOptions.h"
33 #include "llvm/Support/Debug.h"
38 X86FrameLowering::X86FrameLowering(const X86Subtarget &STI,
39 unsigned StackAlignOverride)
40 : TargetFrameLowering(StackGrowsDown, StackAlignOverride,
41 STI.is64Bit() ? -8 : -4),
42 STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) {
43 // Cache a bunch of frame-related predicates for this subtarget.
44 SlotSize = TRI->getSlotSize();
45 Is64Bit = STI.is64Bit();
46 IsLP64 = STI.isTarget64BitLP64();
47 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
48 Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64();
49 StackPtr = TRI->getStackRegister();
52 bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
53 return !MF.getFrameInfo()->hasVarSizedObjects() &&
54 !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences();
57 /// canSimplifyCallFramePseudos - If there is a reserved call frame, the
58 /// call frame pseudos can be simplified. Having a FP, as in the default
59 /// implementation, is not sufficient here since we can't always use it.
60 /// Use a more nuanced condition.
62 X86FrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
63 return hasReservedCallFrame(MF) ||
64 (hasFP(MF) && !TRI->needsStackRealignment(MF)) ||
65 TRI->hasBasePointer(MF);
68 // needsFrameIndexResolution - Do we need to perform FI resolution for
69 // this function. Normally, this is required only when the function
70 // has any stack objects. However, FI resolution actually has another job,
71 // not apparent from the title - it resolves callframesetup/destroy
72 // that were not simplified earlier.
73 // So, this is required for x86 functions that have push sequences even
74 // when there are no stack objects.
76 X86FrameLowering::needsFrameIndexResolution(const MachineFunction &MF) const {
77 return MF.getFrameInfo()->hasStackObjects() ||
78 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences();
81 /// hasFP - Return true if the specified function should have a dedicated frame
82 /// pointer register. This is true if the function has variable sized allocas
83 /// or if frame pointer elimination is disabled.
84 bool X86FrameLowering::hasFP(const MachineFunction &MF) const {
85 const MachineFrameInfo *MFI = MF.getFrameInfo();
86 const MachineModuleInfo &MMI = MF.getMMI();
88 return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
89 TRI->needsStackRealignment(MF) ||
90 MFI->hasVarSizedObjects() ||
91 MFI->isFrameAddressTaken() || MFI->hasOpaqueSPAdjustment() ||
92 MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() ||
93 MMI.callsUnwindInit() || MMI.hasEHFunclets() || MMI.callsEHReturn() ||
94 MFI->hasStackMap() || MFI->hasPatchPoint());
97 static unsigned getSUBriOpcode(unsigned IsLP64, int64_t Imm) {
100 return X86::SUB64ri8;
101 return X86::SUB64ri32;
104 return X86::SUB32ri8;
109 static unsigned getADDriOpcode(unsigned IsLP64, int64_t Imm) {
112 return X86::ADD64ri8;
113 return X86::ADD64ri32;
116 return X86::ADD32ri8;
121 static unsigned getSUBrrOpcode(unsigned isLP64) {
122 return isLP64 ? X86::SUB64rr : X86::SUB32rr;
125 static unsigned getADDrrOpcode(unsigned isLP64) {
126 return isLP64 ? X86::ADD64rr : X86::ADD32rr;
129 static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) {
132 return X86::AND64ri8;
133 return X86::AND64ri32;
136 return X86::AND32ri8;
140 static unsigned getLEArOpcode(unsigned IsLP64) {
141 return IsLP64 ? X86::LEA64r : X86::LEA32r;
144 /// findDeadCallerSavedReg - Return a caller-saved register that isn't live
145 /// when it reaches the "return" instruction. We can then pop a stack object
146 /// to this register without worry about clobbering it.
147 static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB,
148 MachineBasicBlock::iterator &MBBI,
149 const X86RegisterInfo *TRI,
151 const MachineFunction *MF = MBB.getParent();
152 const Function *F = MF->getFunction();
153 if (!F || MF->getMMI().callsEHReturn())
156 const TargetRegisterClass &AvailableRegs = *TRI->getGPRsForTailCall(*MF);
158 unsigned Opc = MBBI->getOpcode();
165 case X86::TCRETURNdi:
166 case X86::TCRETURNri:
167 case X86::TCRETURNmi:
168 case X86::TCRETURNdi64:
169 case X86::TCRETURNri64:
170 case X86::TCRETURNmi64:
172 case X86::EH_RETURN64: {
173 SmallSet<uint16_t, 8> Uses;
174 for (unsigned i = 0, e = MBBI->getNumOperands(); i != e; ++i) {
175 MachineOperand &MO = MBBI->getOperand(i);
176 if (!MO.isReg() || MO.isDef())
178 unsigned Reg = MO.getReg();
181 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
185 for (auto CS : AvailableRegs)
186 if (!Uses.count(CS) && CS != X86::RIP)
194 static bool isEAXLiveIn(MachineFunction &MF) {
195 for (MachineRegisterInfo::livein_iterator II = MF.getRegInfo().livein_begin(),
196 EE = MF.getRegInfo().livein_end(); II != EE; ++II) {
197 unsigned Reg = II->first;
199 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
200 Reg == X86::AH || Reg == X86::AL)
207 /// Check if the flags need to be preserved before the terminators.
208 /// This would be the case, if the eflags is live-in of the region
209 /// composed by the terminators or live-out of that region, without
210 /// being defined by a terminator.
212 flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) {
213 for (const MachineInstr &MI : MBB.terminators()) {
214 for (const MachineOperand &MO : MI.operands()) {
217 unsigned Reg = MO.getReg();
218 if (Reg != X86::EFLAGS)
221 // This terminator needs an eflags that is not defined
222 // by a previous another terminator:
223 // EFLAGS is live-in of the region composed by the terminators.
226 // This terminator defines the eflags, i.e., we don't need to preserve it.
231 // None of the terminators use or define the eflags.
232 // Check if they are live-out, that would imply we need to preserve them.
233 for (const MachineBasicBlock *Succ : MBB.successors())
234 if (Succ->isLiveIn(X86::EFLAGS))
240 /// emitSPUpdate - Emit a series of instructions to increment / decrement the
241 /// stack pointer by a constant value.
242 void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
243 MachineBasicBlock::iterator &MBBI,
244 int64_t NumBytes, bool InEpilogue) const {
245 bool isSub = NumBytes < 0;
246 uint64_t Offset = isSub ? -NumBytes : NumBytes;
248 uint64_t Chunk = (1LL << 31) - 1;
249 DebugLoc DL = MBB.findDebugLoc(MBBI);
252 if (Offset > Chunk) {
253 // Rather than emit a long series of instructions for large offsets,
254 // load the offset into a register and do one sub/add
257 if (isSub && !isEAXLiveIn(*MBB.getParent()))
258 Reg = (unsigned)(Is64Bit ? X86::RAX : X86::EAX);
260 Reg = findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit);
263 unsigned Opc = Is64Bit ? X86::MOV64ri : X86::MOV32ri;
264 BuildMI(MBB, MBBI, DL, TII.get(Opc), Reg)
267 ? getSUBrrOpcode(Is64Bit)
268 : getADDrrOpcode(Is64Bit);
269 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
272 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
278 uint64_t ThisVal = std::min(Offset, Chunk);
279 if (ThisVal == (Is64Bit ? 8 : 4)) {
280 // Use push / pop instead.
282 ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
283 : findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit);
286 ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
287 : (Is64Bit ? X86::POP64r : X86::POP32r);
288 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc))
289 .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub));
291 MI->setFlag(MachineInstr::FrameSetup);
293 MI->setFlag(MachineInstr::FrameDestroy);
299 MachineInstrBuilder MI = BuildStackAdjustment(
300 MBB, MBBI, DL, isSub ? -ThisVal : ThisVal, InEpilogue);
302 MI.setMIFlag(MachineInstr::FrameSetup);
304 MI.setMIFlag(MachineInstr::FrameDestroy);
310 MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
311 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL,
312 int64_t Offset, bool InEpilogue) const {
313 assert(Offset != 0 && "zero offset stack adjustment requested");
315 // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue
319 // Check if inserting the prologue at the beginning
320 // of MBB would require to use LEA operations.
321 // We need to use LEA operations if EFLAGS is live in, because
322 // it means an instruction will read it before it gets defined.
323 UseLEA = STI.useLeaForSP() || MBB.isLiveIn(X86::EFLAGS);
325 // If we can use LEA for SP but we shouldn't, check that none
326 // of the terminators uses the eflags. Otherwise we will insert
327 // a ADD that will redefine the eflags and break the condition.
328 // Alternatively, we could move the ADD, but this may not be possible
329 // and is an optimization anyway.
330 UseLEA = canUseLEAForSPInEpilogue(*MBB.getParent());
331 if (UseLEA && !STI.useLeaForSP())
332 UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB);
333 // If that assert breaks, that means we do not do the right thing
334 // in canUseAsEpilogue.
335 assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) &&
336 "We shouldn't have allowed this insertion point");
339 MachineInstrBuilder MI;
341 MI = addRegOffset(BuildMI(MBB, MBBI, DL,
342 TII.get(getLEArOpcode(Uses64BitFramePtr)),
344 StackPtr, false, Offset);
346 bool IsSub = Offset < 0;
347 uint64_t AbsOffset = IsSub ? -Offset : Offset;
348 unsigned Opc = IsSub ? getSUBriOpcode(Uses64BitFramePtr, AbsOffset)
349 : getADDriOpcode(Uses64BitFramePtr, AbsOffset);
350 MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
353 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
358 int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
359 MachineBasicBlock::iterator &MBBI,
360 bool doMergeWithPrevious) const {
361 if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
362 (!doMergeWithPrevious && MBBI == MBB.end()))
365 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI;
366 MachineBasicBlock::iterator NI = doMergeWithPrevious ? nullptr
368 unsigned Opc = PI->getOpcode();
371 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 ||
372 Opc == X86::ADD32ri || Opc == X86::ADD32ri8 ||
373 Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
374 PI->getOperand(0).getReg() == StackPtr){
375 Offset += PI->getOperand(2).getImm();
377 if (!doMergeWithPrevious) MBBI = NI;
378 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 ||
379 Opc == X86::SUB32ri || Opc == X86::SUB32ri8) &&
380 PI->getOperand(0).getReg() == StackPtr) {
381 Offset -= PI->getOperand(2).getImm();
383 if (!doMergeWithPrevious) MBBI = NI;
389 void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB,
390 MachineBasicBlock::iterator MBBI, DebugLoc DL,
391 MCCFIInstruction CFIInst) const {
392 MachineFunction &MF = *MBB.getParent();
393 unsigned CFIIndex = MF.getMMI().addFrameInst(CFIInst);
394 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
395 .addCFIIndex(CFIIndex);
399 X86FrameLowering::emitCalleeSavedFrameMoves(MachineBasicBlock &MBB,
400 MachineBasicBlock::iterator MBBI,
402 MachineFunction &MF = *MBB.getParent();
403 MachineFrameInfo *MFI = MF.getFrameInfo();
404 MachineModuleInfo &MMI = MF.getMMI();
405 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
407 // Add callee saved registers to move list.
408 const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
409 if (CSI.empty()) return;
411 // Calculate offsets.
412 for (std::vector<CalleeSavedInfo>::const_iterator
413 I = CSI.begin(), E = CSI.end(); I != E; ++I) {
414 int64_t Offset = MFI->getObjectOffset(I->getFrameIdx());
415 unsigned Reg = I->getReg();
417 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
418 BuildCFI(MBB, MBBI, DL,
419 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
423 /// usesTheStack - This function checks if any of the users of EFLAGS
424 /// copies the EFLAGS. We know that the code that lowers COPY of EFLAGS has
425 /// to use the stack, and if we don't adjust the stack we clobber the first
427 /// See X86InstrInfo::copyPhysReg.
428 static bool usesTheStack(const MachineFunction &MF) {
429 const MachineRegisterInfo &MRI = MF.getRegInfo();
431 for (MachineRegisterInfo::reg_instr_iterator
432 ri = MRI.reg_instr_begin(X86::EFLAGS), re = MRI.reg_instr_end();
440 MachineInstr *X86FrameLowering::emitStackProbe(MachineFunction &MF,
441 MachineBasicBlock &MBB,
442 MachineBasicBlock::iterator MBBI,
444 bool InProlog) const {
445 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
446 if (STI.isTargetWindowsCoreCLR()) {
448 return emitStackProbeInlineStub(MF, MBB, MBBI, DL, true);
450 return emitStackProbeInline(MF, MBB, MBBI, DL, false);
453 return emitStackProbeCall(MF, MBB, MBBI, DL, InProlog);
457 void X86FrameLowering::inlineStackProbe(MachineFunction &MF,
458 MachineBasicBlock &PrologMBB) const {
459 const StringRef ChkStkStubSymbol = "__chkstk_stub";
460 MachineInstr *ChkStkStub = nullptr;
462 for (MachineInstr &MI : PrologMBB) {
463 if (MI.isCall() && MI.getOperand(0).isSymbol() &&
464 ChkStkStubSymbol == MI.getOperand(0).getSymbolName()) {
470 if (ChkStkStub != nullptr) {
471 MachineBasicBlock::iterator MBBI = std::next(ChkStkStub->getIterator());
472 assert(std::prev(MBBI).operator==(ChkStkStub) &&
473 "MBBI expected after __chkstk_stub.");
474 DebugLoc DL = PrologMBB.findDebugLoc(MBBI);
475 emitStackProbeInline(MF, PrologMBB, MBBI, DL, true);
476 ChkStkStub->eraseFromParent();
480 MachineInstr *X86FrameLowering::emitStackProbeInline(
481 MachineFunction &MF, MachineBasicBlock &MBB,
482 MachineBasicBlock::iterator MBBI, DebugLoc DL, bool InProlog) const {
483 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
484 assert(STI.is64Bit() && "different expansion needed for 32 bit");
485 assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR");
486 const TargetInstrInfo &TII = *STI.getInstrInfo();
487 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
489 // RAX contains the number of bytes of desired stack adjustment.
490 // The handling here assumes this value has already been updated so as to
491 // maintain stack alignment.
493 // We need to exit with RSP modified by this amount and execute suitable
494 // page touches to notify the OS that we're growing the stack responsibly.
495 // All stack probing must be done without modifying RSP.
501 // Flags, TestReg = CopyReg - SizeReg
502 // FinalReg = !Flags.Ovf ? TestReg : ZeroReg
503 // LimitReg = gs magic thread env access
504 // if FinalReg >= LimitReg goto ContinueMBB
506 // RoundReg = page address of FinalReg
508 // LoopReg = PHI(LimitReg,ProbeReg)
509 // ProbeReg = LoopReg - PageSize
511 // if (ProbeReg > RoundReg) goto LoopMBB
514 // [rest of original MBB]
516 // Set up the new basic blocks
517 MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(LLVM_BB);
518 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
519 MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(LLVM_BB);
521 MachineFunction::iterator MBBIter = std::next(MBB.getIterator());
522 MF.insert(MBBIter, RoundMBB);
523 MF.insert(MBBIter, LoopMBB);
524 MF.insert(MBBIter, ContinueMBB);
526 // Split MBB and move the tail portion down to ContinueMBB.
527 MachineBasicBlock::iterator BeforeMBBI = std::prev(MBBI);
528 ContinueMBB->splice(ContinueMBB->begin(), &MBB, MBBI, MBB.end());
529 ContinueMBB->transferSuccessorsAndUpdatePHIs(&MBB);
531 // Some useful constants
532 const int64_t ThreadEnvironmentStackLimit = 0x10;
533 const int64_t PageSize = 0x1000;
534 const int64_t PageMask = ~(PageSize - 1);
536 // Registers we need. For the normal case we use virtual
537 // registers. For the prolog expansion we use RAX, RCX and RDX.
538 MachineRegisterInfo &MRI = MF.getRegInfo();
539 const TargetRegisterClass *RegClass = &X86::GR64RegClass;
540 const unsigned SizeReg = InProlog ? (unsigned)X86::RAX
541 : MRI.createVirtualRegister(RegClass),
542 ZeroReg = InProlog ? (unsigned)X86::RCX
543 : MRI.createVirtualRegister(RegClass),
544 CopyReg = InProlog ? (unsigned)X86::RDX
545 : MRI.createVirtualRegister(RegClass),
546 TestReg = InProlog ? (unsigned)X86::RDX
547 : MRI.createVirtualRegister(RegClass),
548 FinalReg = InProlog ? (unsigned)X86::RDX
549 : MRI.createVirtualRegister(RegClass),
550 RoundedReg = InProlog ? (unsigned)X86::RDX
551 : MRI.createVirtualRegister(RegClass),
552 LimitReg = InProlog ? (unsigned)X86::RCX
553 : MRI.createVirtualRegister(RegClass),
554 JoinReg = InProlog ? (unsigned)X86::RCX
555 : MRI.createVirtualRegister(RegClass),
556 ProbeReg = InProlog ? (unsigned)X86::RCX
557 : MRI.createVirtualRegister(RegClass);
559 // SP-relative offsets where we can save RCX and RDX.
560 int64_t RCXShadowSlot = 0;
561 int64_t RDXShadowSlot = 0;
563 // If inlining in the prolog, save RCX and RDX.
564 // Future optimization: don't save or restore if not live in.
566 // Compute the offsets. We need to account for things already
567 // pushed onto the stack at this point: return address, frame
568 // pointer (if used), and callee saves.
569 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
570 const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize();
571 const bool HasFP = hasFP(MF);
572 RCXShadowSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0);
573 RDXShadowSlot = RCXShadowSlot + 8;
575 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false,
578 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false,
582 // Not in the prolog. Copy RAX to a virtual reg.
583 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), SizeReg).addReg(X86::RAX);
586 // Add code to MBB to check for overflow and set the new target stack pointer
588 BuildMI(&MBB, DL, TII.get(X86::XOR64rr), ZeroReg)
589 .addReg(ZeroReg, RegState::Undef)
590 .addReg(ZeroReg, RegState::Undef);
591 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), CopyReg).addReg(X86::RSP);
592 BuildMI(&MBB, DL, TII.get(X86::SUB64rr), TestReg)
595 BuildMI(&MBB, DL, TII.get(X86::CMOVB64rr), FinalReg)
599 // FinalReg now holds final stack pointer value, or zero if
600 // allocation would overflow. Compare against the current stack
601 // limit from the thread environment block. Note this limit is the
602 // lowest touched page on the stack, not the point at which the OS
603 // will cause an overflow exception, so this is just an optimization
604 // to avoid unnecessarily touching pages that are below the current
605 // SP but already commited to the stack by the OS.
606 BuildMI(&MBB, DL, TII.get(X86::MOV64rm), LimitReg)
610 .addImm(ThreadEnvironmentStackLimit)
612 BuildMI(&MBB, DL, TII.get(X86::CMP64rr)).addReg(FinalReg).addReg(LimitReg);
613 // Jump if the desired stack pointer is at or above the stack limit.
614 BuildMI(&MBB, DL, TII.get(X86::JAE_1)).addMBB(ContinueMBB);
616 // Add code to roundMBB to round the final stack pointer to a page boundary.
617 BuildMI(RoundMBB, DL, TII.get(X86::AND64ri32), RoundedReg)
620 BuildMI(RoundMBB, DL, TII.get(X86::JMP_1)).addMBB(LoopMBB);
622 // LimitReg now holds the current stack limit, RoundedReg page-rounded
623 // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page
624 // and probe until we reach RoundedReg.
626 BuildMI(LoopMBB, DL, TII.get(X86::PHI), JoinReg)
633 addRegOffset(BuildMI(LoopMBB, DL, TII.get(X86::LEA64r), ProbeReg), JoinReg,
636 // Probe by storing a byte onto the stack.
637 BuildMI(LoopMBB, DL, TII.get(X86::MOV8mi))
644 BuildMI(LoopMBB, DL, TII.get(X86::CMP64rr))
647 BuildMI(LoopMBB, DL, TII.get(X86::JNE_1)).addMBB(LoopMBB);
649 MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI();
651 // If in prolog, restore RDX and RCX.
653 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::MOV64rm),
655 X86::RSP, false, RCXShadowSlot);
656 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::MOV64rm),
658 X86::RSP, false, RDXShadowSlot);
661 // Now that the probing is done, add code to continueMBB to update
662 // the stack pointer for real.
663 BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::SUB64rr), X86::RSP)
667 // Add the control flow edges we need.
668 MBB.addSuccessor(ContinueMBB);
669 MBB.addSuccessor(RoundMBB);
670 RoundMBB->addSuccessor(LoopMBB);
671 LoopMBB->addSuccessor(ContinueMBB);
672 LoopMBB->addSuccessor(LoopMBB);
674 // Mark all the instructions added to the prolog as frame setup.
676 for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) {
677 BeforeMBBI->setFlag(MachineInstr::FrameSetup);
679 for (MachineInstr &MI : *RoundMBB) {
680 MI.setFlag(MachineInstr::FrameSetup);
682 for (MachineInstr &MI : *LoopMBB) {
683 MI.setFlag(MachineInstr::FrameSetup);
685 for (MachineBasicBlock::iterator CMBBI = ContinueMBB->begin();
686 CMBBI != ContinueMBBI; ++CMBBI) {
687 CMBBI->setFlag(MachineInstr::FrameSetup);
691 // Possible TODO: physreg liveness for InProlog case.
696 MachineInstr *X86FrameLowering::emitStackProbeCall(
697 MachineFunction &MF, MachineBasicBlock &MBB,
698 MachineBasicBlock::iterator MBBI, DebugLoc DL, bool InProlog) const {
699 bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
703 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
705 CallOp = X86::CALLpcrel32;
709 if (STI.isTargetCygMing()) {
710 Symbol = "___chkstk_ms";
714 } else if (STI.isTargetCygMing())
719 MachineInstrBuilder CI;
720 MachineBasicBlock::iterator ExpansionMBBI = std::prev(MBBI);
722 // All current stack probes take AX and SP as input, clobber flags, and
723 // preserve all registers. x86_64 probes leave RSP unmodified.
724 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
725 // For the large code model, we have to call through a register. Use R11,
726 // as it is scratch in all supported calling conventions.
727 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::R11)
728 .addExternalSymbol(Symbol);
729 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addReg(X86::R11);
731 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addExternalSymbol(Symbol);
734 unsigned AX = Is64Bit ? X86::RAX : X86::EAX;
735 unsigned SP = Is64Bit ? X86::RSP : X86::ESP;
736 CI.addReg(AX, RegState::Implicit)
737 .addReg(SP, RegState::Implicit)
738 .addReg(AX, RegState::Define | RegState::Implicit)
739 .addReg(SP, RegState::Define | RegState::Implicit)
740 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit);
743 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
744 // themselves. It also does not clobber %rax so we can reuse it when
746 BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64rr), X86::RSP)
752 // Apply the frame setup flag to all inserted instrs.
753 for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI)
754 ExpansionMBBI->setFlag(MachineInstr::FrameSetup);
760 MachineInstr *X86FrameLowering::emitStackProbeInlineStub(
761 MachineFunction &MF, MachineBasicBlock &MBB,
762 MachineBasicBlock::iterator MBBI, DebugLoc DL, bool InProlog) const {
764 assert(InProlog && "ChkStkStub called outside prolog!");
766 BuildMI(MBB, MBBI, DL, TII.get(X86::CALLpcrel32))
767 .addExternalSymbol("__chkstk_stub");
772 static unsigned calculateSetFPREG(uint64_t SPAdjust) {
773 // Win64 ABI has a less restrictive limitation of 240; 128 works equally well
774 // and might require smaller successive adjustments.
775 const uint64_t Win64MaxSEHOffset = 128;
776 uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset);
777 // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode.
778 return SEHFrameOffset & -16;
781 // If we're forcing a stack realignment we can't rely on just the frame
782 // info, we need to know the ABI stack alignment as well in case we
783 // have a call out. Otherwise just make sure we have some alignment - we'll
784 // go with the minimum SlotSize.
785 uint64_t X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const {
786 const MachineFrameInfo *MFI = MF.getFrameInfo();
787 uint64_t MaxAlign = MFI->getMaxAlignment(); // Desired stack alignment.
788 unsigned StackAlign = getStackAlignment();
789 if (MF.getFunction()->hasFnAttribute("stackrealign")) {
791 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
792 else if (MaxAlign < SlotSize)
798 void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
799 MachineBasicBlock::iterator MBBI,
800 DebugLoc DL, unsigned Reg,
801 uint64_t MaxAlign) const {
802 uint64_t Val = -MaxAlign;
803 unsigned AndOp = getANDriOpcode(Uses64BitFramePtr, Val);
804 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AndOp), Reg)
807 .setMIFlag(MachineInstr::FrameSetup);
809 // The EFLAGS implicit def is dead.
810 MI->getOperand(3).setIsDead();
813 /// emitPrologue - Push callee-saved registers onto the stack, which
814 /// automatically adjust the stack pointer. Adjust the stack pointer to allocate
815 /// space for local variables. Also emit labels used by the exception handler to
816 /// generate the exception handling frames.
819 Here's a gist of what gets emitted:
821 ; Establish frame pointer, if needed
824 .cfi_def_cfa_offset 16
825 .cfi_offset %rbp, -16
828 .cfi_def_cfa_register %rbp
830 ; Spill general-purpose registers
831 [for all callee-saved GPRs]
834 .cfi_def_cfa_offset (offset from RETADDR)
837 ; If the required stack alignment > default stack alignment
838 ; rsp needs to be re-aligned. This creates a "re-alignment gap"
839 ; of unknown size in the stack frame.
840 [if stack needs re-alignment]
843 ; Allocate space for locals
844 [if target is Windows and allocated space > 4096 bytes]
845 ; Windows needs special care for allocations larger
848 call ___chkstk_ms/___chkstk
854 .seh_stackalloc (size of XMM spill slots)
855 .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots
860 ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved,
861 ; they may get spilled on any platform, if the current function
862 ; calls @llvm.eh.unwind.init
864 [for all callee-saved XMM registers]
865 movaps %<xmm reg>, -MMM(%rbp)
866 [for all callee-saved XMM registers]
867 .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset)
868 ; i.e. the offset relative to (%rbp - SEHFrameOffset)
870 [for all callee-saved XMM registers]
871 movaps %<xmm reg>, KKK(%rsp)
872 [for all callee-saved XMM registers]
873 .seh_savexmm %<xmm reg>, KKK
877 [if needs base pointer]
879 [if needs to restore base pointer]
884 [for all callee-saved registers]
885 .cfi_offset %<reg>, (offset from %rbp)
887 .cfi_def_cfa_offset (offset from RETADDR)
888 [for all callee-saved registers]
889 .cfi_offset %<reg>, (offset from %rsp)
892 - .seh directives are emitted only for Windows 64 ABI
893 - .cfi directives are emitted for all other ABIs
894 - for 32-bit code, substitute %e?? registers for %r??
897 void X86FrameLowering::emitPrologue(MachineFunction &MF,
898 MachineBasicBlock &MBB) const {
899 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
900 "MF used frame lowering for wrong subtarget");
901 MachineBasicBlock::iterator MBBI = MBB.begin();
902 MachineFrameInfo *MFI = MF.getFrameInfo();
903 const Function *Fn = MF.getFunction();
904 MachineModuleInfo &MMI = MF.getMMI();
905 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
906 uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
907 uint64_t StackSize = MFI->getStackSize(); // Number of bytes to allocate.
908 bool IsFunclet = MBB.isEHFuncletEntry();
909 bool FnHasClrFunclet =
910 MMI.hasEHFunclets() &&
911 classifyEHPersonality(Fn->getPersonalityFn()) == EHPersonality::CoreCLR;
912 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
913 bool HasFP = hasFP(MF);
914 bool IsWin64CC = STI.isCallingConvWin64(Fn->getCallingConv());
915 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
916 bool NeedsWinCFI = IsWin64Prologue && Fn->needsUnwindTableEntry();
918 !IsWin64Prologue && (MMI.hasDebugInfo() || Fn->needsUnwindTableEntry());
919 unsigned FramePtr = TRI->getFrameRegister(MF);
920 const unsigned MachineFramePtr =
921 STI.isTarget64BitILP32()
922 ? getX86SubSuperRegister(FramePtr, MVT::i64, false)
924 unsigned BasePtr = TRI->getBaseRegister();
926 // Debug location must be unknown since the first debug location is used
927 // to determine the end of the prologue.
930 // Add RETADDR move area to callee saved frame size.
931 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
932 if (TailCallReturnAddrDelta && IsWin64Prologue)
933 report_fatal_error("Can't handle guaranteed tail call under win64 yet");
935 if (TailCallReturnAddrDelta < 0)
936 X86FI->setCalleeSavedFrameSize(
937 X86FI->getCalleeSavedFrameSize() - TailCallReturnAddrDelta);
939 bool UseStackProbe = (STI.isOSWindows() && !STI.isTargetMachO());
941 // The default stack probe size is 4096 if the function has no stackprobesize
943 unsigned StackProbeSize = 4096;
944 if (Fn->hasFnAttribute("stack-probe-size"))
945 Fn->getFnAttribute("stack-probe-size")
947 .getAsInteger(0, StackProbeSize);
949 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
950 // function, and use up to 128 bytes of stack space, don't have a frame
951 // pointer, calls, or dynamic alloca then we do not need to adjust the
952 // stack pointer (we fit in the Red Zone). We also check that we don't
953 // push and pop from the stack.
954 if (Is64Bit && !Fn->hasFnAttribute(Attribute::NoRedZone) &&
955 !TRI->needsStackRealignment(MF) &&
956 !MFI->hasVarSizedObjects() && // No dynamic alloca.
957 !MFI->adjustsStack() && // No calls.
958 !IsWin64CC && // Win64 has no Red Zone
959 !usesTheStack(MF) && // Don't push and pop.
960 !MF.shouldSplitStack()) { // Regular stack
961 uint64_t MinSize = X86FI->getCalleeSavedFrameSize();
962 if (HasFP) MinSize += SlotSize;
963 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
964 MFI->setStackSize(StackSize);
967 // Insert stack pointer adjustment for later moving of return addr. Only
968 // applies to tail call optimized functions where the callee argument stack
969 // size is bigger than the callers.
970 if (TailCallReturnAddrDelta < 0) {
971 BuildStackAdjustment(MBB, MBBI, DL, TailCallReturnAddrDelta,
972 /*InEpilogue=*/false)
973 .setMIFlag(MachineInstr::FrameSetup);
976 // Mapping for machine moves:
978 // DST: VirtualFP AND
979 // SRC: VirtualFP => DW_CFA_def_cfa_offset
980 // ELSE => DW_CFA_def_cfa
982 // SRC: VirtualFP AND
983 // DST: Register => DW_CFA_def_cfa_register
986 // OFFSET < 0 => DW_CFA_offset_extended_sf
987 // REG < 64 => DW_CFA_offset + Reg
988 // ELSE => DW_CFA_offset_extended
990 uint64_t NumBytes = 0;
991 int stackGrowth = -SlotSize;
993 // Find the funclet establisher parameter
994 unsigned Establisher = X86::NoRegister;
996 Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX;
998 Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX;
1000 if (IsWin64Prologue && IsFunclet & !IsClrFunclet) {
1001 // Immediately spill establisher into the home slot.
1002 // The runtime cares about this.
1003 // MOV64mr %rdx, 16(%rsp)
1004 unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1005 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MOVmr)), StackPtr, true, 16)
1006 .addReg(Establisher)
1007 .setMIFlag(MachineInstr::FrameSetup);
1008 MBB.addLiveIn(Establisher);
1012 // Calculate required stack adjustment.
1013 uint64_t FrameSize = StackSize - SlotSize;
1014 // If required, include space for extra hidden slot for stashing base pointer.
1015 if (X86FI->getRestoreBasePointer())
1016 FrameSize += SlotSize;
1018 NumBytes = FrameSize - X86FI->getCalleeSavedFrameSize();
1020 // Callee-saved registers are pushed on stack before the stack is realigned.
1021 if (TRI->needsStackRealignment(MF) && !IsWin64Prologue)
1022 NumBytes = RoundUpToAlignment(NumBytes, MaxAlign);
1024 // Get the offset of the stack slot for the EBP register, which is
1025 // guaranteed to be the last slot by processFunctionBeforeFrameFinalized.
1026 // Update the frame offset adjustment.
1028 MFI->setOffsetAdjustment(-NumBytes);
1030 assert(MFI->getOffsetAdjustment() == -(int)NumBytes &&
1031 "should calculate same local variable offset for funclets");
1033 // Save EBP/RBP into the appropriate stack slot.
1034 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
1035 .addReg(MachineFramePtr, RegState::Kill)
1036 .setMIFlag(MachineInstr::FrameSetup);
1038 if (NeedsDwarfCFI) {
1039 // Mark the place where EBP/RBP was saved.
1040 // Define the current CFA rule to use the provided offset.
1042 BuildCFI(MBB, MBBI, DL,
1043 MCCFIInstruction::createDefCfaOffset(nullptr, 2 * stackGrowth));
1045 // Change the rule for the FramePtr to be an "offset" rule.
1046 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
1047 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createOffset(
1048 nullptr, DwarfFramePtr, 2 * stackGrowth));
1052 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1054 .setMIFlag(MachineInstr::FrameSetup);
1057 if (!IsWin64Prologue && !IsFunclet) {
1058 // Update EBP with the new base value.
1059 BuildMI(MBB, MBBI, DL,
1060 TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
1063 .setMIFlag(MachineInstr::FrameSetup);
1065 if (NeedsDwarfCFI) {
1066 // Mark effective beginning of when frame pointer becomes valid.
1067 // Define the current CFA to use the EBP/RBP register.
1068 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
1069 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaRegister(
1070 nullptr, DwarfFramePtr));
1074 // Mark the FramePtr as live-in in every block. Don't do this again for
1075 // funclet prologues.
1077 for (MachineBasicBlock &EveryMBB : MF)
1078 EveryMBB.addLiveIn(MachineFramePtr);
1081 assert(!IsFunclet && "funclets without FPs not yet implemented");
1082 NumBytes = StackSize - X86FI->getCalleeSavedFrameSize();
1085 // For EH funclets, only allocate enough space for outgoing calls. Save the
1086 // NumBytes value that we would've used for the parent frame.
1087 unsigned ParentFrameNumBytes = NumBytes;
1089 NumBytes = getWinEHFuncletFrameSize(MF);
1091 // Skip the callee-saved push instructions.
1092 bool PushedRegs = false;
1093 int StackOffset = 2 * stackGrowth;
1095 while (MBBI != MBB.end() &&
1096 MBBI->getFlag(MachineInstr::FrameSetup) &&
1097 (MBBI->getOpcode() == X86::PUSH32r ||
1098 MBBI->getOpcode() == X86::PUSH64r)) {
1100 unsigned Reg = MBBI->getOperand(0).getReg();
1103 if (!HasFP && NeedsDwarfCFI) {
1104 // Mark callee-saved push instruction.
1105 // Define the current CFA rule to use the provided offset.
1107 BuildCFI(MBB, MBBI, DL,
1108 MCCFIInstruction::createDefCfaOffset(nullptr, StackOffset));
1109 StackOffset += stackGrowth;
1113 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)).addImm(Reg).setMIFlag(
1114 MachineInstr::FrameSetup);
1118 // Realign stack after we pushed callee-saved registers (so that we'll be
1119 // able to calculate their offsets from the frame pointer).
1120 // Don't do this for Win64, it needs to realign the stack after the prologue.
1121 if (!IsWin64Prologue && !IsFunclet && TRI->needsStackRealignment(MF)) {
1122 assert(HasFP && "There should be a frame pointer if stack is realigned.");
1123 BuildStackAlignAND(MBB, MBBI, DL, StackPtr, MaxAlign);
1126 // If there is an SUB32ri of ESP immediately before this instruction, merge
1127 // the two. This can be the case when tail call elimination is enabled and
1128 // the callee has more arguments then the caller.
1129 NumBytes -= mergeSPUpdates(MBB, MBBI, true);
1131 // Adjust stack pointer: ESP -= numbytes.
1133 // Windows and cygwin/mingw require a prologue helper routine when allocating
1134 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw
1135 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the
1136 // stack and adjust the stack pointer in one go. The 64-bit version of
1137 // __chkstk is only responsible for probing the stack. The 64-bit prologue is
1138 // responsible for adjusting the stack pointer. Touching the stack at 4K
1139 // increments is necessary to ensure that the guard pages used by the OS
1140 // virtual memory manager are allocated in correct sequence.
1141 uint64_t AlignedNumBytes = NumBytes;
1142 if (IsWin64Prologue && !IsFunclet && TRI->needsStackRealignment(MF))
1143 AlignedNumBytes = RoundUpToAlignment(AlignedNumBytes, MaxAlign);
1144 if (AlignedNumBytes >= StackProbeSize && UseStackProbe) {
1145 // Check whether EAX is livein for this function.
1146 bool isEAXAlive = isEAXLiveIn(MF);
1149 // Sanity check that EAX is not livein for this function.
1150 // It should not be, so throw an assert.
1151 assert(!Is64Bit && "EAX is livein in x64 case!");
1154 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r))
1155 .addReg(X86::EAX, RegState::Kill)
1156 .setMIFlag(MachineInstr::FrameSetup);
1160 // Handle the 64-bit Windows ABI case where we need to call __chkstk.
1161 // Function prologue is responsible for adjusting the stack pointer.
1162 if (isUInt<32>(NumBytes)) {
1163 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
1165 .setMIFlag(MachineInstr::FrameSetup);
1166 } else if (isInt<32>(NumBytes)) {
1167 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri32), X86::RAX)
1169 .setMIFlag(MachineInstr::FrameSetup);
1171 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::RAX)
1173 .setMIFlag(MachineInstr::FrameSetup);
1176 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
1177 // We'll also use 4 already allocated bytes for EAX.
1178 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
1179 .addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
1180 .setMIFlag(MachineInstr::FrameSetup);
1183 // Call __chkstk, __chkstk_ms, or __alloca.
1184 emitStackProbe(MF, MBB, MBBI, DL, true);
1189 addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm), X86::EAX),
1190 StackPtr, false, NumBytes - 4);
1191 MI->setFlag(MachineInstr::FrameSetup);
1192 MBB.insert(MBBI, MI);
1194 } else if (NumBytes) {
1195 emitSPUpdate(MBB, MBBI, -(int64_t)NumBytes, /*InEpilogue=*/false);
1198 if (NeedsWinCFI && NumBytes)
1199 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc))
1201 .setMIFlag(MachineInstr::FrameSetup);
1203 int SEHFrameOffset = 0;
1204 unsigned SPOrEstablisher;
1207 // The establisher parameter passed to a CLR funclet is actually a pointer
1208 // to the (mostly empty) frame of its nearest enclosing funclet; we have
1209 // to find the root function establisher frame by loading the PSPSym from
1210 // the intermediate frame.
1211 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
1212 MachinePointerInfo NoInfo;
1213 MBB.addLiveIn(Establisher);
1214 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), Establisher),
1215 Establisher, false, PSPSlotOffset)
1216 .addMemOperand(MF.getMachineMemOperand(
1217 NoInfo, MachineMemOperand::MOLoad, SlotSize, SlotSize));
1219 // Save the root establisher back into the current funclet's (mostly
1220 // empty) frame, in case a sub-funclet or the GC needs it.
1221 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr,
1222 false, PSPSlotOffset)
1223 .addReg(Establisher)
1225 MF.getMachineMemOperand(NoInfo, MachineMemOperand::MOStore |
1226 MachineMemOperand::MOVolatile,
1227 SlotSize, SlotSize));
1229 SPOrEstablisher = Establisher;
1231 SPOrEstablisher = StackPtr;
1234 if (IsWin64Prologue && HasFP) {
1235 // Set RBP to a small fixed offset from RSP. In the funclet case, we base
1236 // this calculation on the incoming establisher, which holds the value of
1237 // RSP from the parent frame at the end of the prologue.
1238 SEHFrameOffset = calculateSetFPREG(ParentFrameNumBytes);
1240 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr),
1241 SPOrEstablisher, false, SEHFrameOffset);
1243 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr)
1244 .addReg(SPOrEstablisher);
1246 // If this is not a funclet, emit the CFI describing our frame pointer.
1247 if (NeedsWinCFI && !IsFunclet)
1248 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
1250 .addImm(SEHFrameOffset)
1251 .setMIFlag(MachineInstr::FrameSetup);
1252 } else if (IsFunclet && STI.is32Bit()) {
1253 // Reset EBP / ESI to something good for funclets.
1254 MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL);
1255 // If we're a catch funclet, we can be returned to via catchret. Save ESP
1256 // into the registration node so that the runtime will restore it for us.
1257 if (!MBB.isCleanupFuncletEntry()) {
1258 assert(classifyEHPersonality(Fn->getPersonalityFn()) ==
1259 EHPersonality::MSVC_CXX);
1261 int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex;
1262 int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg);
1263 // ESP is the first field, so no extra displacement is needed.
1264 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32mr)), FrameReg,
1270 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) {
1271 const MachineInstr *FrameInstr = &*MBBI;
1276 if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) {
1277 if (X86::FR64RegClass.contains(Reg)) {
1278 unsigned IgnoredFrameReg;
1279 int Offset = getFrameIndexReference(MF, FI, IgnoredFrameReg);
1280 Offset += SEHFrameOffset;
1282 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM))
1285 .setMIFlag(MachineInstr::FrameSetup);
1292 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue))
1293 .setMIFlag(MachineInstr::FrameSetup);
1295 if (FnHasClrFunclet && !IsFunclet) {
1296 // Save the so-called Initial-SP (i.e. the value of the stack pointer
1297 // immediately after the prolog) into the PSPSlot so that funclets
1298 // and the GC can recover it.
1299 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
1300 auto PSPInfo = MachinePointerInfo::getFixedStack(
1301 MF, MF.getWinEHFuncInfo()->PSPSymFrameIdx);
1302 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, false,
1305 .addMemOperand(MF.getMachineMemOperand(
1306 PSPInfo, MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
1307 SlotSize, SlotSize));
1310 // Realign stack after we spilled callee-saved registers (so that we'll be
1311 // able to calculate their offsets from the frame pointer).
1312 // Win64 requires aligning the stack after the prologue.
1313 if (IsWin64Prologue && TRI->needsStackRealignment(MF)) {
1314 assert(HasFP && "There should be a frame pointer if stack is realigned.");
1315 BuildStackAlignAND(MBB, MBBI, DL, SPOrEstablisher, MaxAlign);
1318 // We already dealt with stack realignment and funclets above.
1319 if (IsFunclet && STI.is32Bit())
1322 // If we need a base pointer, set it up here. It's whatever the value
1323 // of the stack pointer is at this point. Any variable size objects
1324 // will be allocated after this, so we can still use the base pointer
1325 // to reference locals.
1326 if (TRI->hasBasePointer(MF)) {
1327 // Update the base pointer with the current stack pointer.
1328 unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
1329 BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr)
1330 .addReg(SPOrEstablisher)
1331 .setMIFlag(MachineInstr::FrameSetup);
1332 if (X86FI->getRestoreBasePointer()) {
1333 // Stash value of base pointer. Saving RSP instead of EBP shortens
1334 // dependence chain. Used by SjLj EH.
1335 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1336 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)),
1337 FramePtr, true, X86FI->getRestoreBasePointerOffset())
1338 .addReg(SPOrEstablisher)
1339 .setMIFlag(MachineInstr::FrameSetup);
1342 if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) {
1343 // Stash the value of the frame pointer relative to the base pointer for
1344 // Win32 EH. This supports Win32 EH, which does the inverse of the above:
1345 // it recovers the frame pointer from the base pointer rather than the
1346 // other way around.
1347 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1350 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg);
1351 assert(UsedReg == BasePtr);
1352 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), UsedReg, true, Offset)
1354 .setMIFlag(MachineInstr::FrameSetup);
1358 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
1359 // Mark end of stack pointer adjustment.
1360 if (!HasFP && NumBytes) {
1361 // Define the current CFA rule to use the provided offset.
1363 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaOffset(
1364 nullptr, -StackSize + stackGrowth));
1367 // Emit DWARF info specifying the offsets of the callee-saved registers.
1369 emitCalleeSavedFrameMoves(MBB, MBBI, DL);
1373 bool X86FrameLowering::canUseLEAForSPInEpilogue(
1374 const MachineFunction &MF) const {
1375 // We can't use LEA instructions for adjusting the stack pointer if this is a
1376 // leaf function in the Win64 ABI. Only ADD instructions may be used to
1377 // deallocate the stack.
1378 // This means that we can use LEA for SP in two situations:
1379 // 1. We *aren't* using the Win64 ABI which means we are free to use LEA.
1380 // 2. We *have* a frame pointer which means we are permitted to use LEA.
1381 return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || hasFP(MF);
1384 static bool isFuncletReturnInstr(MachineInstr *MI) {
1385 switch (MI->getOpcode()) {
1387 case X86::CLEANUPRET:
1392 llvm_unreachable("impossible");
1395 // CLR funclets use a special "Previous Stack Pointer Symbol" slot on the
1396 // stack. It holds a pointer to the bottom of the root function frame. The
1397 // establisher frame pointer passed to a nested funclet may point to the
1398 // (mostly empty) frame of its parent funclet, but it will need to find
1399 // the frame of the root function to access locals. To facilitate this,
1400 // every funclet copies the pointer to the bottom of the root function
1401 // frame into a PSPSym slot in its own (mostly empty) stack frame. Using the
1402 // same offset for the PSPSym in the root function frame that's used in the
1403 // funclets' frames allows each funclet to dynamically accept any ancestor
1404 // frame as its establisher argument (the runtime doesn't guarantee the
1405 // immediate parent for some reason lost to history), and also allows the GC,
1406 // which uses the PSPSym for some bookkeeping, to find it in any funclet's
1407 // frame with only a single offset reported for the entire method.
1409 X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const {
1410 const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo();
1411 // getFrameIndexReferenceFromSP has an out ref parameter for the stack
1412 // pointer register; pass a dummy that we ignore
1414 int Offset = getFrameIndexReferenceFromSP(MF, Info.PSPSymFrameIdx, SPReg);
1415 assert(Offset >= 0);
1416 return static_cast<unsigned>(Offset);
1420 X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const {
1421 // This is the size of the pushed CSRs.
1423 MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize();
1424 // This is the amount of stack a funclet needs to allocate.
1426 EHPersonality Personality =
1427 classifyEHPersonality(MF.getFunction()->getPersonalityFn());
1428 if (Personality == EHPersonality::CoreCLR) {
1429 // CLR funclets need to hold enough space to include the PSPSym, at the
1430 // same offset from the stack pointer (immediately after the prolog) as it
1431 // resides at in the main function.
1432 UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize;
1434 // Other funclets just need enough stack for outgoing call arguments.
1435 UsedSize = MF.getFrameInfo()->getMaxCallFrameSize();
1437 // RBP is not included in the callee saved register block. After pushing RBP,
1438 // everything is 16 byte aligned. Everything we allocate before an outgoing
1439 // call must also be 16 byte aligned.
1440 unsigned FrameSizeMinusRBP =
1441 RoundUpToAlignment(CSSize + UsedSize, getStackAlignment());
1442 // Subtract out the size of the callee saved registers. This is how much stack
1443 // each funclet will allocate.
1444 return FrameSizeMinusRBP - CSSize;
1447 void X86FrameLowering::emitEpilogue(MachineFunction &MF,
1448 MachineBasicBlock &MBB) const {
1449 const MachineFrameInfo *MFI = MF.getFrameInfo();
1450 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1451 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1453 if (MBBI != MBB.end())
1454 DL = MBBI->getDebugLoc();
1455 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
1456 const bool Is64BitILP32 = STI.isTarget64BitILP32();
1457 unsigned FramePtr = TRI->getFrameRegister(MF);
1458 unsigned MachineFramePtr =
1459 Is64BitILP32 ? getX86SubSuperRegister(FramePtr, MVT::i64, false)
1462 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
1464 IsWin64Prologue && MF.getFunction()->needsUnwindTableEntry();
1465 bool IsFunclet = isFuncletReturnInstr(MBBI);
1466 MachineBasicBlock *TargetMBB = nullptr;
1468 // Get the number of bytes to allocate from the FrameInfo.
1469 uint64_t StackSize = MFI->getStackSize();
1470 uint64_t MaxAlign = calculateMaxStackAlign(MF);
1471 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
1472 uint64_t NumBytes = 0;
1474 if (MBBI->getOpcode() == X86::CATCHRET) {
1475 // SEH shouldn't use catchret.
1476 assert(!isAsynchronousEHPersonality(
1477 classifyEHPersonality(MF.getFunction()->getPersonalityFn())) &&
1478 "SEH should not use CATCHRET");
1480 NumBytes = getWinEHFuncletFrameSize(MF);
1481 assert(hasFP(MF) && "EH funclets without FP not yet implemented");
1482 TargetMBB = MBBI->getOperand(0).getMBB();
1485 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::POP64r : X86::POP32r),
1487 .setMIFlag(MachineInstr::FrameDestroy);
1488 } else if (MBBI->getOpcode() == X86::CLEANUPRET) {
1489 NumBytes = getWinEHFuncletFrameSize(MF);
1490 assert(hasFP(MF) && "EH funclets without FP not yet implemented");
1491 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::POP64r : X86::POP32r),
1493 .setMIFlag(MachineInstr::FrameDestroy);
1494 } else if (hasFP(MF)) {
1495 // Calculate required stack adjustment.
1496 uint64_t FrameSize = StackSize - SlotSize;
1497 NumBytes = FrameSize - CSSize;
1499 // Callee-saved registers were pushed on stack before the stack was
1501 if (TRI->needsStackRealignment(MF) && !IsWin64Prologue)
1502 NumBytes = RoundUpToAlignment(FrameSize, MaxAlign);
1505 BuildMI(MBB, MBBI, DL,
1506 TII.get(Is64Bit ? X86::POP64r : X86::POP32r), MachineFramePtr)
1507 .setMIFlag(MachineInstr::FrameDestroy);
1509 NumBytes = StackSize - CSSize;
1511 uint64_t SEHStackAllocAmt = NumBytes;
1513 // Skip the callee-saved pop instructions.
1514 while (MBBI != MBB.begin()) {
1515 MachineBasicBlock::iterator PI = std::prev(MBBI);
1516 unsigned Opc = PI->getOpcode();
1518 if ((Opc != X86::POP32r || !PI->getFlag(MachineInstr::FrameDestroy)) &&
1519 (Opc != X86::POP64r || !PI->getFlag(MachineInstr::FrameDestroy)) &&
1520 Opc != X86::DBG_VALUE && !PI->isTerminator())
1525 MachineBasicBlock::iterator FirstCSPop = MBBI;
1528 // Fill EAX/RAX with the address of the target block.
1529 unsigned ReturnReg = STI.is64Bit() ? X86::RAX : X86::EAX;
1530 if (STI.is64Bit()) {
1531 // LEA64r TargetMBB(%rip), %rax
1532 BuildMI(MBB, FirstCSPop, DL, TII.get(X86::LEA64r), ReturnReg)
1539 // MOV32ri $TargetMBB, %eax
1540 BuildMI(MBB, FirstCSPop, DL, TII.get(X86::MOV32ri), ReturnReg)
1543 // Record that we've taken the address of TargetMBB and no longer just
1544 // reference it in a terminator.
1545 TargetMBB->setHasAddressTaken();
1548 if (MBBI != MBB.end())
1549 DL = MBBI->getDebugLoc();
1551 // If there is an ADD32ri or SUB32ri of ESP immediately before this
1552 // instruction, merge the two instructions.
1553 if (NumBytes || MFI->hasVarSizedObjects())
1554 NumBytes += mergeSPUpdates(MBB, MBBI, true);
1556 // If dynamic alloca is used, then reset esp to point to the last callee-saved
1557 // slot before popping them off! Same applies for the case, when stack was
1558 // realigned. Don't do this if this was a funclet epilogue, since the funclets
1559 // will not do realignment or dynamic stack allocation.
1560 if ((TRI->needsStackRealignment(MF) || MFI->hasVarSizedObjects()) &&
1562 if (TRI->needsStackRealignment(MF))
1564 unsigned SEHFrameOffset = calculateSetFPREG(SEHStackAllocAmt);
1565 uint64_t LEAAmount =
1566 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
1568 // There are only two legal forms of epilogue:
1569 // - add SEHAllocationSize, %rsp
1570 // - lea SEHAllocationSize(%FramePtr), %rsp
1572 // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence.
1573 // However, we may use this sequence if we have a frame pointer because the
1574 // effects of the prologue can safely be undone.
1575 if (LEAAmount != 0) {
1576 unsigned Opc = getLEArOpcode(Uses64BitFramePtr);
1577 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr),
1578 FramePtr, false, LEAAmount);
1581 unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr);
1582 BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
1586 } else if (NumBytes) {
1587 // Adjust stack pointer back: ESP += numbytes.
1588 emitSPUpdate(MBB, MBBI, NumBytes, /*InEpilogue=*/true);
1592 // Windows unwinder will not invoke function's exception handler if IP is
1593 // either in prologue or in epilogue. This behavior causes a problem when a
1594 // call immediately precedes an epilogue, because the return address points
1595 // into the epilogue. To cope with that, we insert an epilogue marker here,
1596 // then replace it with a 'nop' if it ends up immediately after a CALL in the
1597 // final emitted code.
1599 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue));
1601 // Add the return addr area delta back since we are not tail calling.
1602 int Offset = -1 * X86FI->getTCReturnAddrDelta();
1603 assert(Offset >= 0 && "TCDelta should never be positive");
1605 MBBI = MBB.getFirstTerminator();
1607 // Check for possible merge with preceding ADD instruction.
1608 Offset += mergeSPUpdates(MBB, MBBI, true);
1609 emitSPUpdate(MBB, MBBI, Offset, /*InEpilogue=*/true);
1613 // NOTE: this only has a subset of the full frame index logic. In
1614 // particular, the FI < 0 and AfterFPPop logic is handled in
1615 // X86RegisterInfo::eliminateFrameIndex, but not here. Possibly
1616 // (probably?) it should be moved into here.
1617 int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
1618 unsigned &FrameReg) const {
1619 const MachineFrameInfo *MFI = MF.getFrameInfo();
1621 // We can't calculate offset from frame pointer if the stack is realigned,
1622 // so enforce usage of stack/base pointer. The base pointer is used when we
1623 // have dynamic allocas in addition to dynamic realignment.
1624 if (TRI->hasBasePointer(MF))
1625 FrameReg = TRI->getBaseRegister();
1626 else if (TRI->needsStackRealignment(MF))
1627 FrameReg = TRI->getStackRegister();
1629 FrameReg = TRI->getFrameRegister(MF);
1631 // Offset will hold the offset from the stack pointer at function entry to the
1633 // We need to factor in additional offsets applied during the prologue to the
1634 // frame, base, and stack pointer depending on which is used.
1635 int Offset = MFI->getObjectOffset(FI) - getOffsetOfLocalArea();
1636 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1637 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
1638 uint64_t StackSize = MFI->getStackSize();
1639 bool HasFP = hasFP(MF);
1640 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
1641 int64_t FPDelta = 0;
1643 if (IsWin64Prologue) {
1644 assert(!MFI->hasCalls() || (StackSize % 16) == 8);
1646 // Calculate required stack adjustment.
1647 uint64_t FrameSize = StackSize - SlotSize;
1648 // If required, include space for extra hidden slot for stashing base pointer.
1649 if (X86FI->getRestoreBasePointer())
1650 FrameSize += SlotSize;
1651 uint64_t NumBytes = FrameSize - CSSize;
1653 uint64_t SEHFrameOffset = calculateSetFPREG(NumBytes);
1654 if (FI && FI == X86FI->getFAIndex())
1655 return -SEHFrameOffset;
1657 // FPDelta is the offset from the "traditional" FP location of the old base
1658 // pointer followed by return address and the location required by the
1659 // restricted Win64 prologue.
1660 // Add FPDelta to all offsets below that go through the frame pointer.
1661 FPDelta = FrameSize - SEHFrameOffset;
1662 assert((!MFI->hasCalls() || (FPDelta % 16) == 0) &&
1663 "FPDelta isn't aligned per the Win64 ABI!");
1667 if (TRI->hasBasePointer(MF)) {
1668 assert(HasFP && "VLAs and dynamic stack realign, but no FP?!");
1670 // Skip the saved EBP.
1671 return Offset + SlotSize + FPDelta;
1673 assert((-(Offset + StackSize)) % MFI->getObjectAlignment(FI) == 0);
1674 return Offset + StackSize;
1676 } else if (TRI->needsStackRealignment(MF)) {
1678 // Skip the saved EBP.
1679 return Offset + SlotSize + FPDelta;
1681 assert((-(Offset + StackSize)) % MFI->getObjectAlignment(FI) == 0);
1682 return Offset + StackSize;
1684 // FIXME: Support tail calls
1687 return Offset + StackSize;
1689 // Skip the saved EBP.
1692 // Skip the RETADDR move area
1693 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
1694 if (TailCallReturnAddrDelta < 0)
1695 Offset -= TailCallReturnAddrDelta;
1698 return Offset + FPDelta;
1701 // Simplified from getFrameIndexReference keeping only StackPointer cases
1702 int X86FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
1704 unsigned &FrameReg) const {
1705 const MachineFrameInfo *MFI = MF.getFrameInfo();
1706 // Does not include any dynamic realign.
1707 const uint64_t StackSize = MFI->getStackSize();
1710 // LLVM arranges the stack as follows:
1715 // PUSH RBP <-- RBP points here
1717 // ~~~~~~~ <-- possible stack realignment (non-win64)
1720 // ... <-- RSP after prologue points here
1721 // ~~~~~~~ <-- possible stack realignment (win64)
1723 // if (hasVarSizedObjects()):
1724 // ... <-- "base pointer" (ESI/RBX) points here
1726 // ... <-- RSP points here
1728 // Case 1: In the simple case of no stack realignment and no dynamic
1729 // allocas, both "fixed" stack objects (arguments and CSRs) are addressable
1730 // with fixed offsets from RSP.
1732 // Case 2: In the case of stack realignment with no dynamic allocas, fixed
1733 // stack objects are addressed with RBP and regular stack objects with RSP.
1735 // Case 3: In the case of dynamic allocas and stack realignment, RSP is used
1736 // to address stack arguments for outgoing calls and nothing else. The "base
1737 // pointer" points to local variables, and RBP points to fixed objects.
1739 // In cases 2 and 3, we can only answer for non-fixed stack objects, and the
1740 // answer we give is relative to the SP after the prologue, and not the
1741 // SP in the middle of the function.
1743 assert((!MFI->isFixedObjectIndex(FI) || !TRI->needsStackRealignment(MF) ||
1744 STI.isTargetWin64()) &&
1745 "offset from fixed object to SP is not static");
1747 // We don't handle tail calls, and shouldn't be seeing them either.
1748 int TailCallReturnAddrDelta =
1749 MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta();
1750 assert(!(TailCallReturnAddrDelta < 0) && "we don't handle this case!");
1754 // Fill in FrameReg output argument.
1755 FrameReg = TRI->getStackRegister();
1757 // This is how the math works out:
1759 // %rsp grows (i.e. gets lower) left to right. Each box below is
1760 // one word (eight bytes). Obj0 is the stack slot we're trying to
1763 // ----------------------------------
1764 // | BP | Obj0 | Obj1 | ... | ObjN |
1765 // ----------------------------------
1769 // A is the incoming stack pointer.
1770 // (B - A) is the local area offset (-8 for x86-64) [1]
1771 // (C - A) is the Offset returned by MFI->getObjectOffset for Obj0 [2]
1773 // |(E - B)| is the StackSize (absolute value, positive). For a
1774 // stack that grown down, this works out to be (B - E). [3]
1776 // E is also the value of %rsp after stack has been set up, and we
1777 // want (C - E) -- the value we can add to %rsp to get to Obj0. Now
1778 // (C - E) == (C - A) - (B - A) + (B - E)
1779 // { Using [1], [2] and [3] above }
1780 // == getObjectOffset - LocalAreaOffset + StackSize
1783 // Get the Offset from the StackPointer
1784 int Offset = MFI->getObjectOffset(FI) - getOffsetOfLocalArea();
1786 return Offset + StackSize;
1789 bool X86FrameLowering::assignCalleeSavedSpillSlots(
1790 MachineFunction &MF, const TargetRegisterInfo *TRI,
1791 std::vector<CalleeSavedInfo> &CSI) const {
1792 MachineFrameInfo *MFI = MF.getFrameInfo();
1793 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1795 unsigned CalleeSavedFrameSize = 0;
1796 int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta();
1799 // emitPrologue always spills frame register the first thing.
1800 SpillSlotOffset -= SlotSize;
1801 MFI->CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
1803 // Since emitPrologue and emitEpilogue will handle spilling and restoring of
1804 // the frame register, we can delete it from CSI list and not have to worry
1805 // about avoiding it later.
1806 unsigned FPReg = TRI->getFrameRegister(MF);
1807 for (unsigned i = 0; i < CSI.size(); ++i) {
1808 if (TRI->regsOverlap(CSI[i].getReg(),FPReg)) {
1809 CSI.erase(CSI.begin() + i);
1815 // Assign slots for GPRs. It increases frame size.
1816 for (unsigned i = CSI.size(); i != 0; --i) {
1817 unsigned Reg = CSI[i - 1].getReg();
1819 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
1822 SpillSlotOffset -= SlotSize;
1823 CalleeSavedFrameSize += SlotSize;
1825 int SlotIndex = MFI->CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
1826 CSI[i - 1].setFrameIdx(SlotIndex);
1829 X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize);
1831 // Assign slots for XMMs.
1832 for (unsigned i = CSI.size(); i != 0; --i) {
1833 unsigned Reg = CSI[i - 1].getReg();
1834 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
1837 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1839 SpillSlotOffset -= std::abs(SpillSlotOffset) % RC->getAlignment();
1841 SpillSlotOffset -= RC->getSize();
1843 MFI->CreateFixedSpillStackObject(RC->getSize(), SpillSlotOffset);
1844 CSI[i - 1].setFrameIdx(SlotIndex);
1845 MFI->ensureMaxAlignment(RC->getAlignment());
1851 bool X86FrameLowering::spillCalleeSavedRegisters(
1852 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1853 const std::vector<CalleeSavedInfo> &CSI,
1854 const TargetRegisterInfo *TRI) const {
1855 DebugLoc DL = MBB.findDebugLoc(MI);
1857 // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI
1858 // for us, and there are no XMM CSRs on Win32.
1859 if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows())
1862 // Push GPRs. It increases frame size.
1863 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
1864 for (unsigned i = CSI.size(); i != 0; --i) {
1865 unsigned Reg = CSI[i - 1].getReg();
1867 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
1869 // Add the callee-saved register as live-in. It's killed at the spill.
1872 BuildMI(MBB, MI, DL, TII.get(Opc)).addReg(Reg, RegState::Kill)
1873 .setMIFlag(MachineInstr::FrameSetup);
1876 // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
1877 // It can be done by spilling XMMs to stack frame.
1878 for (unsigned i = CSI.size(); i != 0; --i) {
1879 unsigned Reg = CSI[i-1].getReg();
1880 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
1882 // Add the callee-saved register as live-in. It's killed at the spill.
1884 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1886 TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i - 1].getFrameIdx(), RC,
1889 MI->setFlag(MachineInstr::FrameSetup);
1896 bool X86FrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
1897 MachineBasicBlock::iterator MI,
1898 const std::vector<CalleeSavedInfo> &CSI,
1899 const TargetRegisterInfo *TRI) const {
1903 if (isFuncletReturnInstr(MI) && STI.isOSWindows()) {
1904 // Don't restore CSRs in 32-bit EH funclets. Matches
1905 // spillCalleeSavedRegisters.
1908 // Don't restore CSRs before an SEH catchret. SEH except blocks do not form
1909 // funclets. emitEpilogue transforms these to normal jumps.
1910 if (MI->getOpcode() == X86::CATCHRET) {
1911 const Function *Func = MBB.getParent()->getFunction();
1912 bool IsSEH = isAsynchronousEHPersonality(
1913 classifyEHPersonality(Func->getPersonalityFn()));
1919 DebugLoc DL = MBB.findDebugLoc(MI);
1921 // Reload XMMs from stack frame.
1922 for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
1923 unsigned Reg = CSI[i].getReg();
1924 if (X86::GR64RegClass.contains(Reg) ||
1925 X86::GR32RegClass.contains(Reg))
1928 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1929 TII.loadRegFromStackSlot(MBB, MI, Reg, CSI[i].getFrameIdx(), RC, TRI);
1933 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r;
1934 for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
1935 unsigned Reg = CSI[i].getReg();
1936 if (!X86::GR64RegClass.contains(Reg) &&
1937 !X86::GR32RegClass.contains(Reg))
1940 BuildMI(MBB, MI, DL, TII.get(Opc), Reg)
1941 .setMIFlag(MachineInstr::FrameDestroy);
1946 void X86FrameLowering::determineCalleeSaves(MachineFunction &MF,
1947 BitVector &SavedRegs,
1948 RegScavenger *RS) const {
1949 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1951 MachineFrameInfo *MFI = MF.getFrameInfo();
1953 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1954 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
1956 if (TailCallReturnAddrDelta < 0) {
1957 // create RETURNADDR area
1966 MFI->CreateFixedObject(-TailCallReturnAddrDelta,
1967 TailCallReturnAddrDelta - SlotSize, true);
1970 // Spill the BasePtr if it's used.
1971 if (TRI->hasBasePointer(MF)) {
1972 SavedRegs.set(TRI->getBaseRegister());
1974 // Allocate a spill slot for EBP if we have a base pointer and EH funclets.
1975 if (MF.getMMI().hasEHFunclets()) {
1976 int FI = MFI->CreateSpillStackObject(SlotSize, SlotSize);
1977 X86FI->setHasSEHFramePtrSave(true);
1978 X86FI->setSEHFramePtrSaveIndex(FI);
1984 HasNestArgument(const MachineFunction *MF) {
1985 const Function *F = MF->getFunction();
1986 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
1988 if (I->hasNestAttr())
1994 /// GetScratchRegister - Get a temp register for performing work in the
1995 /// segmented stack and the Erlang/HiPE stack prologue. Depending on platform
1996 /// and the properties of the function either one or two registers will be
1997 /// needed. Set primary to true for the first register, false for the second.
1999 GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary) {
2000 CallingConv::ID CallingConvention = MF.getFunction()->getCallingConv();
2003 if (CallingConvention == CallingConv::HiPE) {
2005 return Primary ? X86::R14 : X86::R13;
2007 return Primary ? X86::EBX : X86::EDI;
2012 return Primary ? X86::R11 : X86::R12;
2014 return Primary ? X86::R11D : X86::R12D;
2017 bool IsNested = HasNestArgument(&MF);
2019 if (CallingConvention == CallingConv::X86_FastCall ||
2020 CallingConvention == CallingConv::Fast) {
2022 report_fatal_error("Segmented stacks does not support fastcall with "
2023 "nested function.");
2024 return Primary ? X86::EAX : X86::ECX;
2027 return Primary ? X86::EDX : X86::EAX;
2028 return Primary ? X86::ECX : X86::EAX;
2031 // The stack limit in the TCB is set to this many bytes above the actual stack
2033 static const uint64_t kSplitStackAvailable = 256;
2035 void X86FrameLowering::adjustForSegmentedStacks(
2036 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
2037 MachineFrameInfo *MFI = MF.getFrameInfo();
2039 unsigned TlsReg, TlsOffset;
2042 unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true);
2043 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
2044 "Scratch register is live-in");
2046 if (MF.getFunction()->isVarArg())
2047 report_fatal_error("Segmented stacks do not support vararg functions.");
2048 if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() &&
2049 !STI.isTargetWin64() && !STI.isTargetFreeBSD() &&
2050 !STI.isTargetDragonFly())
2051 report_fatal_error("Segmented stacks not supported on this platform.");
2053 // Eventually StackSize will be calculated by a link-time pass; which will
2054 // also decide whether checking code needs to be injected into this particular
2056 StackSize = MFI->getStackSize();
2058 // Do not generate a prologue for functions with a stack of size zero
2062 MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock();
2063 MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock();
2064 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2065 bool IsNested = false;
2067 // We need to know if the function has a nest argument only in 64 bit mode.
2069 IsNested = HasNestArgument(&MF);
2071 // The MOV R10, RAX needs to be in a different block, since the RET we emit in
2072 // allocMBB needs to be last (terminating) instruction.
2074 for (const auto &LI : PrologueMBB.liveins()) {
2075 allocMBB->addLiveIn(LI);
2076 checkMBB->addLiveIn(LI);
2080 allocMBB->addLiveIn(IsLP64 ? X86::R10 : X86::R10D);
2082 MF.push_front(allocMBB);
2083 MF.push_front(checkMBB);
2085 // When the frame size is less than 256 we just compare the stack
2086 // boundary directly to the value of the stack pointer, per gcc.
2087 bool CompareStackPointer = StackSize < kSplitStackAvailable;
2089 // Read the limit off the current stacklet off the stack_guard location.
2091 if (STI.isTargetLinux()) {
2093 TlsOffset = IsLP64 ? 0x70 : 0x40;
2094 } else if (STI.isTargetDarwin()) {
2096 TlsOffset = 0x60 + 90*8; // See pthread_machdep.h. Steal TLS slot 90.
2097 } else if (STI.isTargetWin64()) {
2099 TlsOffset = 0x28; // pvArbitrary, reserved for application use
2100 } else if (STI.isTargetFreeBSD()) {
2103 } else if (STI.isTargetDragonFly()) {
2105 TlsOffset = 0x20; // use tls_tcb.tcb_segstack
2107 report_fatal_error("Segmented stacks not supported on this platform.");
2110 if (CompareStackPointer)
2111 ScratchReg = IsLP64 ? X86::RSP : X86::ESP;
2113 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::LEA64r : X86::LEA64_32r), ScratchReg).addReg(X86::RSP)
2114 .addImm(1).addReg(0).addImm(-StackSize).addReg(0);
2116 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::CMP64rm : X86::CMP32rm)).addReg(ScratchReg)
2117 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg);
2119 if (STI.isTargetLinux()) {
2122 } else if (STI.isTargetDarwin()) {
2124 TlsOffset = 0x48 + 90*4;
2125 } else if (STI.isTargetWin32()) {
2127 TlsOffset = 0x14; // pvArbitrary, reserved for application use
2128 } else if (STI.isTargetDragonFly()) {
2130 TlsOffset = 0x10; // use tls_tcb.tcb_segstack
2131 } else if (STI.isTargetFreeBSD()) {
2132 report_fatal_error("Segmented stacks not supported on FreeBSD i386.");
2134 report_fatal_error("Segmented stacks not supported on this platform.");
2137 if (CompareStackPointer)
2138 ScratchReg = X86::ESP;
2140 BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg).addReg(X86::ESP)
2141 .addImm(1).addReg(0).addImm(-StackSize).addReg(0);
2143 if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64() ||
2144 STI.isTargetDragonFly()) {
2145 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)).addReg(ScratchReg)
2146 .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg);
2147 } else if (STI.isTargetDarwin()) {
2149 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register.
2150 unsigned ScratchReg2;
2152 if (CompareStackPointer) {
2153 // The primary scratch register is available for holding the TLS offset.
2154 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, true);
2155 SaveScratch2 = false;
2157 // Need to use a second register to hold the TLS offset
2158 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, false);
2160 // Unfortunately, with fastcc the second scratch register may hold an
2162 SaveScratch2 = MF.getRegInfo().isLiveIn(ScratchReg2);
2165 // If Scratch2 is live-in then it needs to be saved.
2166 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) &&
2167 "Scratch register is live-in and not saved");
2170 BuildMI(checkMBB, DL, TII.get(X86::PUSH32r))
2171 .addReg(ScratchReg2, RegState::Kill);
2173 BuildMI(checkMBB, DL, TII.get(X86::MOV32ri), ScratchReg2)
2175 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm))
2177 .addReg(ScratchReg2).addImm(1).addReg(0)
2182 BuildMI(checkMBB, DL, TII.get(X86::POP32r), ScratchReg2);
2186 // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
2187 // It jumps to normal execution of the function body.
2188 BuildMI(checkMBB, DL, TII.get(X86::JA_1)).addMBB(&PrologueMBB);
2190 // On 32 bit we first push the arguments size and then the frame size. On 64
2191 // bit, we pass the stack frame size in r10 and the argument size in r11.
2193 // Functions with nested arguments use R10, so it needs to be saved across
2194 // the call to _morestack
2196 const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX;
2197 const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D;
2198 const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D;
2199 const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr;
2200 const unsigned MOVri = IsLP64 ? X86::MOV64ri : X86::MOV32ri;
2203 BuildMI(allocMBB, DL, TII.get(MOVrr), RegAX).addReg(Reg10);
2205 BuildMI(allocMBB, DL, TII.get(MOVri), Reg10)
2207 BuildMI(allocMBB, DL, TII.get(MOVri), Reg11)
2208 .addImm(X86FI->getArgumentStackSize());
2210 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
2211 .addImm(X86FI->getArgumentStackSize());
2212 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
2216 // __morestack is in libgcc
2217 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
2218 // Under the large code model, we cannot assume that __morestack lives
2219 // within 2^31 bytes of the call site, so we cannot use pc-relative
2220 // addressing. We cannot perform the call via a temporary register,
2221 // as the rax register may be used to store the static chain, and all
2222 // other suitable registers may be either callee-save or used for
2223 // parameter passing. We cannot use the stack at this point either
2224 // because __morestack manipulates the stack directly.
2226 // To avoid these issues, perform an indirect call via a read-only memory
2227 // location containing the address.
2229 // This solution is not perfect, as it assumes that the .rodata section
2230 // is laid out within 2^31 bytes of each function body, but this seems
2231 // to be sufficient for JIT.
2232 BuildMI(allocMBB, DL, TII.get(X86::CALL64m))
2236 .addExternalSymbol("__morestack_addr")
2238 MF.getMMI().setUsesMorestackAddr(true);
2241 BuildMI(allocMBB, DL, TII.get(X86::CALL64pcrel32))
2242 .addExternalSymbol("__morestack");
2244 BuildMI(allocMBB, DL, TII.get(X86::CALLpcrel32))
2245 .addExternalSymbol("__morestack");
2249 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET_RESTORE_R10));
2251 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET));
2253 allocMBB->addSuccessor(&PrologueMBB);
2255 checkMBB->addSuccessor(allocMBB);
2256 checkMBB->addSuccessor(&PrologueMBB);
2263 /// Erlang programs may need a special prologue to handle the stack size they
2264 /// might need at runtime. That is because Erlang/OTP does not implement a C
2265 /// stack but uses a custom implementation of hybrid stack/heap architecture.
2266 /// (for more information see Eric Stenman's Ph.D. thesis:
2267 /// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf)
2270 /// temp0 = sp - MaxStack
2271 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
2275 /// call inc_stack # doubles the stack space
2276 /// temp0 = sp - MaxStack
2277 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
2278 void X86FrameLowering::adjustForHiPEPrologue(
2279 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
2280 MachineFrameInfo *MFI = MF.getFrameInfo();
2282 // HiPE-specific values
2283 const unsigned HipeLeafWords = 24;
2284 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5;
2285 const unsigned Guaranteed = HipeLeafWords * SlotSize;
2286 unsigned CallerStkArity = MF.getFunction()->arg_size() > CCRegisteredArgs ?
2287 MF.getFunction()->arg_size() - CCRegisteredArgs : 0;
2288 unsigned MaxStack = MFI->getStackSize() + CallerStkArity*SlotSize + SlotSize;
2290 assert(STI.isTargetLinux() &&
2291 "HiPE prologue is only supported on Linux operating systems.");
2293 // Compute the largest caller's frame that is needed to fit the callees'
2294 // frames. This 'MaxStack' is computed from:
2296 // a) the fixed frame size, which is the space needed for all spilled temps,
2297 // b) outgoing on-stack parameter areas, and
2298 // c) the minimum stack space this function needs to make available for the
2299 // functions it calls (a tunable ABI property).
2300 if (MFI->hasCalls()) {
2301 unsigned MoreStackForCalls = 0;
2303 for (MachineFunction::iterator MBBI = MF.begin(), MBBE = MF.end();
2304 MBBI != MBBE; ++MBBI)
2305 for (MachineBasicBlock::iterator MI = MBBI->begin(), ME = MBBI->end();
2310 // Get callee operand.
2311 const MachineOperand &MO = MI->getOperand(0);
2313 // Only take account of global function calls (no closures etc.).
2317 const Function *F = dyn_cast<Function>(MO.getGlobal());
2321 // Do not update 'MaxStack' for primitive and built-in functions
2322 // (encoded with names either starting with "erlang."/"bif_" or not
2323 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an
2324 // "_", such as the BIF "suspend_0") as they are executed on another
2326 if (F->getName().find("erlang.") != StringRef::npos ||
2327 F->getName().find("bif_") != StringRef::npos ||
2328 F->getName().find_first_of("._") == StringRef::npos)
2331 unsigned CalleeStkArity =
2332 F->arg_size() > CCRegisteredArgs ? F->arg_size()-CCRegisteredArgs : 0;
2333 if (HipeLeafWords - 1 > CalleeStkArity)
2334 MoreStackForCalls = std::max(MoreStackForCalls,
2335 (HipeLeafWords - 1 - CalleeStkArity) * SlotSize);
2337 MaxStack += MoreStackForCalls;
2340 // If the stack frame needed is larger than the guaranteed then runtime checks
2341 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
2342 if (MaxStack > Guaranteed) {
2343 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
2344 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
2346 for (const auto &LI : PrologueMBB.liveins()) {
2347 stackCheckMBB->addLiveIn(LI);
2348 incStackMBB->addLiveIn(LI);
2351 MF.push_front(incStackMBB);
2352 MF.push_front(stackCheckMBB);
2354 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
2355 unsigned LEAop, CMPop, CALLop;
2359 LEAop = X86::LEA64r;
2360 CMPop = X86::CMP64rm;
2361 CALLop = X86::CALL64pcrel32;
2362 SPLimitOffset = 0x90;
2366 LEAop = X86::LEA32r;
2367 CMPop = X86::CMP32rm;
2368 CALLop = X86::CALLpcrel32;
2369 SPLimitOffset = 0x4c;
2372 ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true);
2373 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
2374 "HiPE prologue scratch register is live-in");
2376 // Create new MBB for StackCheck:
2377 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(LEAop), ScratchReg),
2378 SPReg, false, -MaxStack);
2379 // SPLimitOffset is in a fixed heap location (pointed by BP).
2380 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop))
2381 .addReg(ScratchReg), PReg, false, SPLimitOffset);
2382 BuildMI(stackCheckMBB, DL, TII.get(X86::JAE_1)).addMBB(&PrologueMBB);
2384 // Create new MBB for IncStack:
2385 BuildMI(incStackMBB, DL, TII.get(CALLop)).
2386 addExternalSymbol("inc_stack_0");
2387 addRegOffset(BuildMI(incStackMBB, DL, TII.get(LEAop), ScratchReg),
2388 SPReg, false, -MaxStack);
2389 addRegOffset(BuildMI(incStackMBB, DL, TII.get(CMPop))
2390 .addReg(ScratchReg), PReg, false, SPLimitOffset);
2391 BuildMI(incStackMBB, DL, TII.get(X86::JLE_1)).addMBB(incStackMBB);
2393 stackCheckMBB->addSuccessor(&PrologueMBB, {99, 100});
2394 stackCheckMBB->addSuccessor(incStackMBB, {1, 100});
2395 incStackMBB->addSuccessor(&PrologueMBB, {99, 100});
2396 incStackMBB->addSuccessor(incStackMBB, {1, 100});
2403 bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB,
2404 MachineBasicBlock::iterator MBBI, DebugLoc DL, int Offset) const {
2409 if (Offset % SlotSize)
2412 int NumPops = Offset / SlotSize;
2413 // This is only worth it if we have at most 2 pops.
2414 if (NumPops != 1 && NumPops != 2)
2417 // Handle only the trivial case where the adjustment directly follows
2418 // a call. This is the most common one, anyway.
2419 if (MBBI == MBB.begin())
2421 MachineBasicBlock::iterator Prev = std::prev(MBBI);
2422 if (!Prev->isCall() || !Prev->getOperand(1).isRegMask())
2426 unsigned FoundRegs = 0;
2428 auto RegMask = Prev->getOperand(1);
2431 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
2432 // Try to find up to NumPops free registers.
2433 for (auto Candidate : RegClass) {
2435 // Poor man's liveness:
2436 // Since we're immediately after a call, any register that is clobbered
2437 // by the call and not defined by it can be considered dead.
2438 if (!RegMask.clobbersPhysReg(Candidate))
2442 for (const MachineOperand &MO : Prev->implicit_operands()) {
2443 if (MO.isReg() && MO.isDef() && MO.getReg() == Candidate) {
2452 Regs[FoundRegs++] = Candidate;
2453 if (FoundRegs == (unsigned)NumPops)
2460 // If we found only one free register, but need two, reuse the same one twice.
2461 while (FoundRegs < (unsigned)NumPops)
2462 Regs[FoundRegs++] = Regs[0];
2464 for (int i = 0; i < NumPops; ++i)
2465 BuildMI(MBB, MBBI, DL,
2466 TII.get(STI.is64Bit() ? X86::POP64r : X86::POP32r), Regs[i]);
2471 void X86FrameLowering::
2472 eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
2473 MachineBasicBlock::iterator I) const {
2474 bool reserveCallFrame = hasReservedCallFrame(MF);
2475 unsigned Opcode = I->getOpcode();
2476 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
2477 DebugLoc DL = I->getDebugLoc();
2478 uint64_t Amount = !reserveCallFrame ? I->getOperand(0).getImm() : 0;
2479 uint64_t InternalAmt = (isDestroy || Amount) ? I->getOperand(1).getImm() : 0;
2482 if (!reserveCallFrame) {
2483 // If the stack pointer can be changed after prologue, turn the
2484 // adjcallstackup instruction into a 'sub ESP, <amt>' and the
2485 // adjcallstackdown instruction into 'add ESP, <amt>'
2487 // We need to keep the stack aligned properly. To do this, we round the
2488 // amount of space needed for the outgoing arguments up to the next
2489 // alignment boundary.
2490 unsigned StackAlign = getStackAlignment();
2491 Amount = RoundUpToAlignment(Amount, StackAlign);
2493 MachineModuleInfo &MMI = MF.getMMI();
2494 const Function *Fn = MF.getFunction();
2495 bool WindowsCFI = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
2496 bool DwarfCFI = !WindowsCFI &&
2497 (MMI.hasDebugInfo() || Fn->needsUnwindTableEntry());
2499 // If we have any exception handlers in this function, and we adjust
2500 // the SP before calls, we may need to indicate this to the unwinder
2501 // using GNU_ARGS_SIZE. Note that this may be necessary even when
2502 // Amount == 0, because the preceding function may have set a non-0
2504 // TODO: We don't need to reset this between subsequent functions,
2505 // if it didn't change.
2506 bool HasDwarfEHHandlers = !WindowsCFI &&
2507 !MF.getMMI().getLandingPads().empty();
2509 if (HasDwarfEHHandlers && !isDestroy &&
2510 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences())
2511 BuildCFI(MBB, I, DL,
2512 MCCFIInstruction::createGnuArgsSize(nullptr, Amount));
2517 // Factor out the amount that gets handled inside the sequence
2518 // (Pushes of argument for frame setup, callee pops for frame destroy)
2519 Amount -= InternalAmt;
2521 // If this is a callee-pop calling convention, and we're emitting precise
2522 // SP-based CFI, emit a CFA adjust for the amount the callee popped.
2523 if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF) &&
2524 MMI.usePreciseUnwindInfo())
2525 BuildCFI(MBB, I, DL,
2526 MCCFIInstruction::createAdjustCfaOffset(nullptr, -InternalAmt));
2529 // Add Amount to SP to destroy a frame, and subtract to setup.
2530 int Offset = isDestroy ? Amount : -Amount;
2532 if (!(Fn->optForMinSize() &&
2533 adjustStackWithPops(MBB, I, DL, Offset)))
2534 BuildStackAdjustment(MBB, I, DL, Offset, /*InEpilogue=*/false);
2537 if (DwarfCFI && !hasFP(MF)) {
2538 // If we don't have FP, but need to generate unwind information,
2539 // we need to set the correct CFA offset after the stack adjustment.
2540 // How much we adjust the CFA offset depends on whether we're emitting
2541 // CFI only for EH purposes or for debugging. EH only requires the CFA
2542 // offset to be correct at each call site, while for debugging we want
2543 // it to be more precise.
2544 int CFAOffset = Amount;
2545 if (!MMI.usePreciseUnwindInfo())
2546 CFAOffset += InternalAmt;
2547 CFAOffset = isDestroy ? -CFAOffset : CFAOffset;
2548 BuildCFI(MBB, I, DL,
2549 MCCFIInstruction::createAdjustCfaOffset(nullptr, CFAOffset));
2555 if (isDestroy && InternalAmt) {
2556 // If we are performing frame pointer elimination and if the callee pops
2557 // something off the stack pointer, add it back. We do this until we have
2558 // more advanced stack pointer tracking ability.
2559 // We are not tracking the stack pointer adjustment by the callee, so make
2560 // sure we restore the stack pointer immediately after the call, there may
2561 // be spill code inserted between the CALL and ADJCALLSTACKUP instructions.
2562 MachineBasicBlock::iterator B = MBB.begin();
2563 while (I != B && !std::prev(I)->isCall())
2565 BuildStackAdjustment(MBB, I, DL, -InternalAmt, /*InEpilogue=*/false);
2569 bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
2570 assert(MBB.getParent() && "Block is not attached to a function!");
2572 // Win64 has strict requirements in terms of epilogue and we are
2573 // not taking a chance at messing with them.
2574 // I.e., unless this block is already an exit block, we can't use
2575 // it as an epilogue.
2576 if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock())
2579 if (canUseLEAForSPInEpilogue(*MBB.getParent()))
2582 // If we cannot use LEA to adjust SP, we may need to use ADD, which
2583 // clobbers the EFLAGS. Check that we do not need to preserve it,
2584 // otherwise, conservatively assume this is not
2585 // safe to insert the epilogue here.
2586 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
2589 MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers(
2590 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2591 DebugLoc DL, bool RestoreSP) const {
2592 assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env");
2593 assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32");
2594 assert(STI.is32Bit() && !Uses64BitFramePtr &&
2595 "restoring EBP/ESI on non-32-bit target");
2597 MachineFunction &MF = *MBB.getParent();
2598 unsigned FramePtr = TRI->getFrameRegister(MF);
2599 unsigned BasePtr = TRI->getBaseRegister();
2600 WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo();
2601 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2602 MachineFrameInfo *MFI = MF.getFrameInfo();
2604 // FIXME: Don't set FrameSetup flag in catchret case.
2606 int FI = FuncInfo.EHRegNodeFrameIndex;
2607 int EHRegSize = MFI->getObjectSize(FI);
2610 // MOV32rm -EHRegSize(%ebp), %esp
2611 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), X86::ESP),
2612 X86::EBP, true, -EHRegSize)
2613 .setMIFlag(MachineInstr::FrameSetup);
2617 int EHRegOffset = getFrameIndexReference(MF, FI, UsedReg);
2618 int EndOffset = -EHRegOffset - EHRegSize;
2619 FuncInfo.EHRegNodeEndOffset = EndOffset;
2621 if (UsedReg == FramePtr) {
2622 // ADD $offset, %ebp
2623 unsigned ADDri = getADDriOpcode(false, EndOffset);
2624 BuildMI(MBB, MBBI, DL, TII.get(ADDri), FramePtr)
2627 .setMIFlag(MachineInstr::FrameSetup)
2630 assert(EndOffset >= 0 &&
2631 "end of registration object above normal EBP position!");
2632 } else if (UsedReg == BasePtr) {
2633 // LEA offset(%ebp), %esi
2634 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA32r), BasePtr),
2635 FramePtr, false, EndOffset)
2636 .setMIFlag(MachineInstr::FrameSetup);
2637 // MOV32rm SavedEBPOffset(%esi), %ebp
2638 assert(X86FI->getHasSEHFramePtrSave());
2640 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg);
2641 assert(UsedReg == BasePtr);
2642 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), FramePtr),
2643 UsedReg, true, Offset)
2644 .setMIFlag(MachineInstr::FrameSetup);
2646 llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr");
2651 unsigned X86FrameLowering::getWinEHParentFrameOffset(const MachineFunction &MF) const {
2652 // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue.
2653 unsigned Offset = 16;
2654 // RBP is immediately pushed.
2656 // All callee-saved registers are then pushed.
2657 Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize();
2658 // Every funclet allocates enough stack space for the largest outgoing call.
2659 Offset += getWinEHFuncletFrameSize(MF);
2663 void X86FrameLowering::processFunctionBeforeFrameFinalized(
2664 MachineFunction &MF, RegScavenger *RS) const {
2665 // If this function isn't doing Win64-style C++ EH, we don't need to do
2667 const Function *Fn = MF.getFunction();
2668 if (!STI.is64Bit() || !MF.getMMI().hasEHFunclets() ||
2669 classifyEHPersonality(Fn->getPersonalityFn()) != EHPersonality::MSVC_CXX)
2672 // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset
2673 // relative to RSP after the prologue. Find the offset of the last fixed
2674 // object, so that we can allocate a slot immediately following it. If there
2675 // were no fixed objects, use offset -SlotSize, which is immediately after the
2676 // return address. Fixed objects have negative frame indices.
2677 MachineFrameInfo *MFI = MF.getFrameInfo();
2678 int64_t MinFixedObjOffset = -SlotSize;
2679 for (int I = MFI->getObjectIndexBegin(); I < 0; ++I)
2680 MinFixedObjOffset = std::min(MinFixedObjOffset, MFI->getObjectOffset(I));
2682 int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize;
2684 MFI->CreateFixedObject(SlotSize, UnwindHelpOffset, /*Immutable=*/false);
2685 MF.getWinEHFuncInfo()->UnwindHelpFrameIdx = UnwindHelpFI;
2687 // Store -2 into UnwindHelp on function entry. We have to scan forwards past
2688 // other frame setup instructions.
2689 MachineBasicBlock &MBB = MF.front();
2690 auto MBBI = MBB.begin();
2691 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
2694 DebugLoc DL = MBB.findDebugLoc(MBBI);
2695 addFrameReference(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mi32)),