X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FTarget%2FX86%2FX86FloatingPoint.cpp;h=6f1d8e5237329d01a4f7315d30c571406ea994bb;hb=cf0db29df20d9c665da7e82bb261bdd7cf7f1b2b;hp=141cde27b25cbd5cab51e853752d74c89e84bbfb;hpb=b720be6a50f4e1b3280d2b029ee38dda14577525;p=oota-llvm.git diff --git a/lib/Target/X86/X86FloatingPoint.cpp b/lib/Target/X86/X86FloatingPoint.cpp index 141cde27b25..6f1d8e52373 100644 --- a/lib/Target/X86/X86FloatingPoint.cpp +++ b/lib/Target/X86/X86FloatingPoint.cpp @@ -23,32 +23,39 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "x86-codegen" #include "X86.h" #include "X86InstrInfo.h" -#include "llvm/InlineAsm.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/EdgeBundles.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/IR/InlineAsm.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetSubtargetInfo.h" #include +#include using namespace llvm; +#define DEBUG_TYPE "x86-codegen" + STATISTIC(NumFXCH, "Number of fxch instructions inserted"); STATISTIC(NumFP , "Number of floating point instructions"); namespace { + const unsigned ScratchFPReg = 7; + struct FPS : public MachineFunctionPass { static char ID; FPS() : MachineFunctionPass(ID) { @@ -59,7 +66,7 @@ namespace { memset(RegMap, 0, sizeof(RegMap)); } - virtual void getAnalysisUsage(AnalysisUsage &AU) const { + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); AU.addRequired(); AU.addPreservedID(MachineLoopInfoID); @@ -67,9 +74,9 @@ namespace { MachineFunctionPass::getAnalysisUsage(AU); } - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; - virtual const char *getPassName() const { return "X86 FP Stackifier"; } + const char *getPassName() const override { return "X86 FP Stackifier"; } private: const TargetInstrInfo *TII; // Machine instruction info. @@ -111,13 +118,14 @@ namespace { EdgeBundles *Bundles; // Return a bitmask of FP registers in block's live-in list. - unsigned calcLiveInMask(MachineBasicBlock *MBB) { + static unsigned calcLiveInMask(MachineBasicBlock *MBB) { unsigned Mask = 0; for (MachineBasicBlock::livein_iterator I = MBB->livein_begin(), E = MBB->livein_end(); I != E; ++I) { - unsigned Reg = *I - X86::FP0; - if (Reg < 8) - Mask |= 1 << Reg; + unsigned Reg = *I; + if (Reg < X86::FP0 || Reg > X86::FP6) + continue; + Mask |= 1 << (Reg - X86::FP0); } return Mask; } @@ -135,7 +143,7 @@ namespace { unsigned StackTop; // The current top of the FP stack. enum { - NumFPRegs = 16 // Including scratch pseudo-registers. + NumFPRegs = 8 // Including scratch pseudo-registers. }; // For each live FP register, point to its Stack[] entry. @@ -144,27 +152,6 @@ namespace { // register allocator thinks. unsigned RegMap[NumFPRegs]; - // Pending fixed registers - Inline assembly needs FP registers to appear - // in fixed stack slot positions. This is handled by copying FP registers - // to ST registers before the instruction, and copying back after the - // instruction. - // - // This is modeled with pending ST registers. NumPendingSTs is the number - // of ST registers (ST0-STn) we are tracking. PendingST[n] points to an FP - // register that holds the ST value. The ST registers are not moved into - // place until immediately before the instruction that needs them. - // - // It can happen that we need an ST register to be live when no FP register - // holds the value: - // - // %ST0 = COPY %FP4 - // - // When that happens, we allocate a scratch FP register to hold the ST - // value. That means every register in PendingST must be live. - - unsigned NumPendingSTs; - unsigned char PendingST[8]; - // Set up our stack model to match the incoming registers to MBB. void setupBlockStack(); @@ -178,9 +165,6 @@ namespace { dbgs() << " FP" << Stack[i]; assert(RegMap[Stack[i]] == i && "Stack[] doesn't match RegMap[]!"); } - for (unsigned i = 0; i != NumPendingSTs; ++i) - dbgs() << ", ST" << i << " in FP" << unsigned(PendingST[i]); - dbgs() << "\n"; } #endif @@ -197,19 +181,6 @@ namespace { return Slot < StackTop && Stack[Slot] == RegNo; } - /// getScratchReg - Return an FP register that is not currently in use. - unsigned getScratchReg() { - for (int i = NumFPRegs - 1; i >= 8; --i) - if (!isLive(i)) - return i; - llvm_unreachable("Ran out of scratch FP registers"); - } - - /// isScratchReg - Returns trus if RegNo is a scratch FP register. - bool isScratchReg(unsigned RegNo) { - return RegNo > 8 && RegNo < NumFPRegs; - } - /// getStackEntry - Return the X86::FP register in register ST(i). unsigned getStackEntry(unsigned STi) const { if (STi >= StackTop) @@ -261,21 +232,6 @@ namespace { BuildMI(*MBB, I, dl, TII->get(X86::LD_Frr)).addReg(STReg); } - /// duplicatePendingSTBeforeKill - The instruction at I is about to kill - /// RegNo. If any PendingST registers still need the RegNo value, duplicate - /// them to new scratch registers. - void duplicatePendingSTBeforeKill(unsigned RegNo, MachineInstr *I) { - for (unsigned i = 0; i != NumPendingSTs; ++i) { - if (PendingST[i] != RegNo) - continue; - unsigned SR = getScratchReg(); - DEBUG(dbgs() << "Duplicating pending ST" << i - << " in FP" << RegNo << " to FP" << SR << '\n'); - duplicateToTop(RegNo, SR, I); - PendingST[i] = SR; - } - } - /// popStackAfter - Pop the current value off of the top of the FP stack /// after the specified instruction. void popStackAfter(MachineBasicBlock::iterator &I); @@ -302,6 +258,7 @@ namespace { bool processBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB); + void handleCall(MachineBasicBlock::iterator &I); void handleZeroArgFP(MachineBasicBlock::iterator &I); void handleOneArgFP(MachineBasicBlock::iterator &I); void handleOneArgFPRW(MachineBasicBlock::iterator &I); @@ -311,16 +268,18 @@ namespace { void handleSpecialFP(MachineBasicBlock::iterator &I); // Check if a COPY instruction is using FP registers. - bool isFPCopy(MachineInstr *MI) { + static bool isFPCopy(MachineInstr *MI) { unsigned DstReg = MI->getOperand(0).getReg(); unsigned SrcReg = MI->getOperand(1).getReg(); return X86::RFP80RegClass.contains(DstReg) || X86::RFP80RegClass.contains(SrcReg); } + + void setKillFlags(MachineBasicBlock &MBB) const; }; char FPS::ID = 0; -} +} // namespace FunctionPass *llvm::createX86FloatingPointStackifierPass() { return new FPS(); } @@ -341,7 +300,7 @@ bool FPS::runOnMachineFunction(MachineFunction &MF) { // function. If it is all integer, there is nothing for us to do! bool FPIsUsed = false; - assert(X86::FP6 == X86::FP0+6 && "Register enums aren't sorted right!"); + static_assert(X86::FP6 == X86::FP0+6, "Register enums aren't sorted right!"); for (unsigned i = 0; i <= 6; ++i) if (MF.getRegInfo().isPhysRegUsed(X86::FP0+i)) { FPIsUsed = true; @@ -352,7 +311,7 @@ bool FPS::runOnMachineFunction(MachineFunction &MF) { if (!FPIsUsed) return false; Bundles = &getAnalysis(); - TII = MF.getTarget().getInstrInfo(); + TII = MF.getSubtarget().getInstrInfo(); // Prepare cross-MBB liveness. bundleCFG(MF); @@ -365,15 +324,13 @@ bool FPS::runOnMachineFunction(MachineFunction &MF) { MachineBasicBlock *Entry = MF.begin(); bool Changed = false; - for (df_ext_iterator > - I = df_ext_begin(Entry, Processed), E = df_ext_end(Entry, Processed); - I != E; ++I) - Changed |= processBasicBlock(MF, **I); + for (MachineBasicBlock *BB : depth_first_ext(Entry, Processed)) + Changed |= processBasicBlock(MF, *BB); // Process any unreachable blocks in arbitrary order now. if (MF.size() != Processed.size()) for (MachineFunction::iterator BB = MF.begin(), E = MF.end(); BB != E; ++BB) - if (Processed.insert(BB)) + if (Processed.insert(BB).second) Changed |= processBasicBlock(MF, *BB); LiveBundles.clear(); @@ -407,8 +364,8 @@ void FPS::bundleCFG(MachineFunction &MF) { bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) { bool Changed = false; MBB = &BB; - NumPendingSTs = 0; + setKillFlags(BB); setupBlockStack(); for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) { @@ -426,12 +383,15 @@ bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) { X86::RFP80RegClass.contains(MI->getOperand(0).getReg())) FPInstClass = X86II::SpecialFP; + if (MI->isCall()) + FPInstClass = X86II::SpecialFP; + if (FPInstClass == X86II::NotFP) continue; // Efficiently ignore non-fp insts! - MachineInstr *PrevMI = 0; + MachineInstr *PrevMI = nullptr; if (I != BB.begin()) - PrevMI = prior(I); + PrevMI = std::prev(I); ++NumFP; // Keep track of # of pseudo instrs DEBUG(dbgs() << "\nFPInst:\t" << *MI); @@ -460,7 +420,9 @@ bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) { // after definition. If so, pop them. for (unsigned i = 0, e = DeadRegs.size(); i != e; ++i) { unsigned Reg = DeadRegs[i]; - if (Reg >= X86::FP0 && Reg <= X86::FP6) { + // Check if Reg is live on the stack. An inline-asm register operand that + // is in the clobber list and marked dead might not be live on the stack. + if (Reg >= X86::FP0 && Reg <= X86::FP6 && isLive(Reg-X86::FP0)) { DEBUG(dbgs() << "Register FP#" << Reg-X86::FP0 << " is dead!\n"); freeStackSlotAfter(I, Reg-X86::FP0); } @@ -474,10 +436,10 @@ bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) { } else { MachineBasicBlock::iterator Start = I; // Rewind to first instruction newly inserted. - while (Start != BB.begin() && prior(Start) != PrevI) --Start; + while (Start != BB.begin() && std::prev(Start) != PrevI) --Start; dbgs() << "Inserted instructions:\n\t"; - Start->print(dbgs(), &MF.getTarget()); - while (++Start != llvm::next(I)) {} + Start->print(dbgs()); + while (++Start != std::next(I)) {} } dumpStack(); ); @@ -577,12 +539,12 @@ namespace { friend bool operator<(const TableEntry &TE, unsigned V) { return TE.from < V; } - friend bool LLVM_ATTRIBUTE_USED operator<(unsigned V, - const TableEntry &TE) { + friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned V, + const TableEntry &TE) { return V < TE.from; } }; -} +} // namespace #ifndef NDEBUG static bool TableIsSorted(const TableEntry *Table, unsigned NumEntries) { @@ -872,7 +834,9 @@ FPS::freeStackSlotBefore(MachineBasicBlock::iterator I, unsigned FPRegNo) { RegMap[TopReg] = OldSlot; RegMap[FPRegNo] = ~0; Stack[--StackTop] = ~0; - return BuildMI(*MBB, I, DebugLoc(), TII->get(X86::ST_FPrr)).addReg(STReg); + return BuildMI(*MBB, I, DebugLoc(), TII->get(X86::ST_FPrr)) + .addReg(STReg) + .getInstr(); } /// adjustLiveRegs - Kill and revive registers such that exactly the FP @@ -893,8 +857,8 @@ void FPS::adjustLiveRegs(unsigned Mask, MachineBasicBlock::iterator I) { // Produce implicit-defs for free by using killed registers. while (Kills && Defs) { - unsigned KReg = CountTrailingZeros_32(Kills); - unsigned DReg = CountTrailingZeros_32(Defs); + unsigned KReg = countTrailingZeros(Kills); + unsigned DReg = countTrailingZeros(Defs); DEBUG(dbgs() << "Renaming %FP" << KReg << " as imp %FP" << DReg << "\n"); std::swap(Stack[getSlot(KReg)], Stack[getSlot(DReg)]); std::swap(RegMap[KReg], RegMap[DReg]); @@ -904,7 +868,7 @@ void FPS::adjustLiveRegs(unsigned Mask, MachineBasicBlock::iterator I) { // Kill registers by popping. if (Kills && I != MBB->begin()) { - MachineBasicBlock::iterator I2 = llvm::prior(I); + MachineBasicBlock::iterator I2 = std::prev(I); while (StackTop) { unsigned KReg = getStackEntry(0); if (!(Kills & (1 << KReg))) @@ -917,7 +881,7 @@ void FPS::adjustLiveRegs(unsigned Mask, MachineBasicBlock::iterator I) { // Manually kill the rest. while (Kills) { - unsigned KReg = CountTrailingZeros_32(Kills); + unsigned KReg = countTrailingZeros(Kills); DEBUG(dbgs() << "Killing %FP" << KReg << "\n"); freeStackSlotBefore(I, KReg); Kills &= ~(1 << KReg); @@ -925,7 +889,7 @@ void FPS::adjustLiveRegs(unsigned Mask, MachineBasicBlock::iterator I) { // Load zeros for all the imp-defs. while(Defs) { - unsigned DReg = CountTrailingZeros_32(Defs); + unsigned DReg = countTrailingZeros(Defs); DEBUG(dbgs() << "Defining %FP" << DReg << " as 0\n"); BuildMI(*MBB, I, DebugLoc(), TII->get(X86::LD_F0)); pushReg(DReg); @@ -934,7 +898,7 @@ void FPS::adjustLiveRegs(unsigned Mask, MachineBasicBlock::iterator I) { // Now we should have the correct registers live. DEBUG(dumpStack()); - assert(StackTop == CountPopulation_32(Mask) && "Live count mismatch"); + assert(StackTop == countPopulation(Mask) && "Live count mismatch"); } /// shuffleStackTop - emit fxch instructions before I to shuffle the top @@ -964,6 +928,31 @@ void FPS::shuffleStackTop(const unsigned char *FixStack, // Instruction transformation implementation //===----------------------------------------------------------------------===// +void FPS::handleCall(MachineBasicBlock::iterator &I) { + unsigned STReturns = 0; + + for (const auto &MO : I->operands()) { + if (!MO.isReg()) + continue; + + unsigned R = MO.getReg() - X86::FP0; + + if (R < 8) { + assert(MO.isDef() && MO.isImplicit()); + STReturns |= 1 << R; + } + } + + unsigned N = countTrailingOnes(STReturns); + + // FP registers used for function return must be consecutive starting at + // FP0. + assert(STReturns == 0 || (isMask_32(STReturns) && N <= 2)); + + for (unsigned I = 0; I < N; ++I) + pushReg(N - I - 1); +} + /// handleZeroArgFP - ST(0) = fld0 ST(0) = flds /// void FPS::handleZeroArgFP(MachineBasicBlock::iterator &I) { @@ -990,9 +979,6 @@ void FPS::handleOneArgFP(MachineBasicBlock::iterator &I) { unsigned Reg = getFPReg(MI->getOperand(NumOps-1)); bool KillsSrc = MI->killsRegister(X86::FP0+Reg); - if (KillsSrc) - duplicatePendingSTBeforeKill(Reg, I); - // FISTP64m is strange because there isn't a non-popping versions. // If we have one _and_ we don't want to pop the operand, duplicate the value // on the stack instead of moving it. This ensure that popping the value is @@ -1013,7 +999,7 @@ void FPS::handleOneArgFP(MachineBasicBlock::iterator &I) { MI->getOpcode() == X86::ISTT_Fp32m80 || MI->getOpcode() == X86::ISTT_Fp64m80 || MI->getOpcode() == X86::ST_FpP80m)) { - duplicateToTop(Reg, getScratchReg(), I); + duplicateToTop(Reg, ScratchFPReg, I); } else { moveToTop(Reg, I); // Move to the top of the stack... } @@ -1056,7 +1042,6 @@ void FPS::handleOneArgFPRW(MachineBasicBlock::iterator &I) { bool KillsSrc = MI->killsRegister(X86::FP0+Reg); if (KillsSrc) { - duplicatePendingSTBeforeKill(Reg, I); // If this is the last use of the source register, just make sure it's on // the top of the stack. moveToTop(Reg, I); @@ -1312,71 +1297,22 @@ void FPS::handleCondMovFP(MachineBasicBlock::iterator &I) { /// floating point instructions. This is primarily intended for use by pseudo /// instructions. /// -void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { - MachineInstr *MI = I; +void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) { + MachineInstr *MI = Inst; + + if (MI->isCall()) { + handleCall(Inst); + return; + } + switch (MI->getOpcode()) { default: llvm_unreachable("Unknown SpecialFP instruction!"); case TargetOpcode::COPY: { // We handle three kinds of copies: FP <- FP, FP <- ST, and ST <- FP. const MachineOperand &MO1 = MI->getOperand(1); const MachineOperand &MO0 = MI->getOperand(0); - unsigned DstST = MO0.getReg() - X86::ST0; - unsigned SrcST = MO1.getReg() - X86::ST0; bool KillsSrc = MI->killsRegister(MO1.getReg()); - // ST = COPY FP. Set up a pending ST register. - if (DstST < 8) { - unsigned SrcFP = getFPReg(MO1); - assert(isLive(SrcFP) && "Cannot copy dead register"); - assert(!MO0.isDead() && "Cannot copy to dead ST register"); - - // Unallocated STs are marked as the nonexistent FP255. - while (NumPendingSTs <= DstST) - PendingST[NumPendingSTs++] = NumFPRegs; - - // STi could still be live from a previous inline asm. - if (isScratchReg(PendingST[DstST])) { - DEBUG(dbgs() << "Clobbering old ST in FP" << unsigned(PendingST[DstST]) - << '\n'); - freeStackSlotBefore(MI, PendingST[DstST]); - } - - // When the source is killed, allocate a scratch FP register. - if (KillsSrc) { - duplicatePendingSTBeforeKill(SrcFP, I); - unsigned Slot = getSlot(SrcFP); - unsigned SR = getScratchReg(); - PendingST[DstST] = SR; - Stack[Slot] = SR; - RegMap[SR] = Slot; - } else - PendingST[DstST] = SrcFP; - break; - } - - // FP = COPY ST. Extract fixed stack value. - // Any instruction defining ST registers must have assigned them to a - // scratch register. - if (SrcST < 8) { - unsigned DstFP = getFPReg(MO0); - assert(!isLive(DstFP) && "Cannot copy ST to live FP register"); - assert(NumPendingSTs > SrcST && "Cannot copy from dead ST register"); - unsigned SrcFP = PendingST[SrcST]; - assert(isScratchReg(SrcFP) && "Expected ST in a scratch register"); - assert(isLive(SrcFP) && "Scratch holding ST is dead"); - - // DstFP steals the stack slot from SrcFP. - unsigned Slot = getSlot(SrcFP); - Stack[Slot] = DstFP; - RegMap[DstFP] = Slot; - - // Always treat the ST as killed. - PendingST[SrcST] = NumFPRegs; - while (NumPendingSTs && PendingST[NumPendingSTs - 1] == NumFPRegs) - --NumPendingSTs; - break; - } - // FP <- FP copy. unsigned DstFP = getFPReg(MO0); unsigned SrcFP = getFPReg(MO1); @@ -1390,7 +1326,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { } else { // For COPY we just duplicate the specified value to a new stack slot. // This could be made better, but would require substantial changes. - duplicateToTop(SrcFP, DstFP, I); + duplicateToTop(SrcFP, DstFP, Inst); } break; } @@ -1399,41 +1335,11 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { // All FP registers must be explicitly defined, so load a 0 instead. unsigned Reg = MI->getOperand(0).getReg() - X86::FP0; DEBUG(dbgs() << "Emitting LD_F0 for implicit FP" << Reg << '\n'); - BuildMI(*MBB, I, MI->getDebugLoc(), TII->get(X86::LD_F0)); + BuildMI(*MBB, Inst, MI->getDebugLoc(), TII->get(X86::LD_F0)); pushReg(Reg); break; } - case X86::FpPOP_RETVAL: { - // The FpPOP_RETVAL instruction is used after calls that return a value on - // the floating point stack. We cannot model this with ST defs since CALL - // instructions have fixed clobber lists. This instruction is interpreted - // to mean that there is one more live register on the stack than we - // thought. - // - // This means that StackTop does not match the hardware stack between a - // call and the FpPOP_RETVAL instructions. We do tolerate FP instructions - // between CALL and FpPOP_RETVAL as long as they don't overflow the - // hardware stack. - unsigned DstFP = getFPReg(MI->getOperand(0)); - - // Move existing stack elements up to reflect reality. - assert(StackTop < 8 && "Stack overflowed before FpPOP_RETVAL"); - if (StackTop) { - std::copy_backward(Stack, Stack + StackTop, Stack + StackTop + 1); - for (unsigned i = 0; i != NumFPRegs; ++i) - ++RegMap[i]; - } - ++StackTop; - - // DstFP is the new bottom of the stack. - Stack[0] = DstFP; - RegMap[DstFP] = 0; - - // DstFP will be killed by processBasicBlock if this was a dead def. - break; - } - case TargetOpcode::INLINEASM: { // The inline asm MachineInstr currently only *uses* FP registers for the // 'f' constraint. These should be turned into the current ST(x) register @@ -1470,19 +1376,30 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { // only tell clobbers from defs by looking at the asm descriptor. unsigned STUses = 0, STDefs = 0, STClobbers = 0, STDeadDefs = 0; unsigned NumOps = 0; + SmallSet FRegIdx; + unsigned RCID; + for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands(); i != e && MI->getOperand(i).isImm(); i += 1 + NumOps) { unsigned Flags = MI->getOperand(i).getImm(); + NumOps = InlineAsm::getNumOperandRegisters(Flags); if (NumOps != 1) continue; const MachineOperand &MO = MI->getOperand(i + 1); if (!MO.isReg()) continue; - unsigned STReg = MO.getReg() - X86::ST0; + unsigned STReg = MO.getReg() - X86::FP0; if (STReg >= 8) continue; + // If the flag has a register class constraint, this must be an operand + // with constraint "f". Record its index and continue. + if (InlineAsm::hasRegClassConstraint(Flags, RCID)) { + FRegIdx.insert(i + 1); + continue; + } + switch (InlineAsm::getKind(Flags)) { case InlineAsm::Kind_RegUse: STUses |= (1u << STReg); @@ -1503,14 +1420,14 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { if (STUses && !isMask_32(STUses)) MI->emitError("fixed input regs must be last on the x87 stack"); - unsigned NumSTUses = CountTrailingOnes_32(STUses); + unsigned NumSTUses = countTrailingOnes(STUses); // Defs must be contiguous from the stack top. ST0-STn. if (STDefs && !isMask_32(STDefs)) { MI->emitError("output regs must be last on the x87 stack"); STDefs = NextPowerOf2(STDefs) - 1; } - unsigned NumSTDefs = CountTrailingOnes_32(STDefs); + unsigned NumSTDefs = countTrailingOnes(STDefs); // So must the clobbered stack slots. ST0-STm, m >= n. if (STClobbers && !isMask_32(STDefs | STClobbers)) @@ -1520,76 +1437,47 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { unsigned STPopped = STUses & (STDefs | STClobbers); if (STPopped && !isMask_32(STPopped)) MI->emitError("implicitly popped regs must be last on the x87 stack"); - unsigned NumSTPopped = CountTrailingOnes_32(STPopped); + unsigned NumSTPopped = countTrailingOnes(STPopped); DEBUG(dbgs() << "Asm uses " << NumSTUses << " fixed regs, pops " << NumSTPopped << ", and defines " << NumSTDefs << " regs.\n"); - // Scan the instruction for FP uses corresponding to "f" constraints. - // Collect FP registers to kill afer the instruction. - // Always kill all the scratch regs. +#ifndef NDEBUG + // If any input operand uses constraint "f", all output register + // constraints must be early-clobber defs. + for (unsigned I = 0, E = MI->getNumOperands(); I < E; ++I) + if (FRegIdx.count(I)) { + assert((1 << getFPReg(MI->getOperand(I)) & STDefs) == 0 && + "Operands with constraint \"f\" cannot overlap with defs"); + } +#endif + + // Collect all FP registers (register operands with constraints "t", "u", + // and "f") to kill afer the instruction. unsigned FPKills = ((1u << NumFPRegs) - 1) & ~0xff; - unsigned FPUsed = 0; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &Op = MI->getOperand(i); if (!Op.isReg() || Op.getReg() < X86::FP0 || Op.getReg() > X86::FP6) continue; - if (!Op.isUse()) - MI->emitError("illegal \"f\" output constraint"); unsigned FPReg = getFPReg(Op); - FPUsed |= 1U << FPReg; // If we kill this operand, make sure to pop it from the stack after the // asm. We just remember it for now, and pop them all off at the end in // a batch. - if (Op.isKill()) + if (Op.isUse() && Op.isKill()) FPKills |= 1U << FPReg; } - // The popped inputs will be killed by the instruction, so duplicate them - // if the FP register needs to be live after the instruction, or if it is - // used in the instruction itself. We effectively treat the popped inputs - // as early clobbers. - for (unsigned i = 0; i < NumSTPopped; ++i) { - if ((FPKills & ~FPUsed) & (1u << PendingST[i])) - continue; - unsigned SR = getScratchReg(); - duplicateToTop(PendingST[i], SR, I); - DEBUG(dbgs() << "Duplicating ST" << i << " in FP" - << unsigned(PendingST[i]) << " to avoid clobbering it.\n"); - PendingST[i] = SR; - } - - // Make sure we have a unique live register for every fixed use. Some of - // them could be undef uses, and we need to emit LD_F0 instructions. - for (unsigned i = 0; i < NumSTUses; ++i) { - if (i < NumPendingSTs && PendingST[i] < NumFPRegs) { - // Check for shared assignments. - for (unsigned j = 0; j < i; ++j) { - if (PendingST[j] != PendingST[i]) - continue; - // STi and STj are inn the same register, create a copy. - unsigned SR = getScratchReg(); - duplicateToTop(PendingST[i], SR, I); - DEBUG(dbgs() << "Duplicating ST" << i << " in FP" - << unsigned(PendingST[i]) - << " to avoid collision with ST" << j << '\n'); - PendingST[i] = SR; - } - continue; - } - unsigned SR = getScratchReg(); - DEBUG(dbgs() << "Emitting LD_F0 for ST" << i << " in FP" << SR << '\n'); - BuildMI(*MBB, I, MI->getDebugLoc(), TII->get(X86::LD_F0)); - pushReg(SR); - PendingST[i] = SR; - if (NumPendingSTs == i) - ++NumPendingSTs; - } - assert(NumPendingSTs >= NumSTUses && "Fixed registers should be assigned"); + // Do not include registers that are implicitly popped by defs/clobbers. + FPKills &= ~(STDefs | STClobbers); // Now we can rearrange the live registers to match what was requested. - shuffleStackTop(PendingST, NumPendingSTs, I); + unsigned char STUsesArray[8]; + + for (unsigned I = 0; I < NumSTUses; ++I) + STUsesArray[I] = I; + + shuffleStackTop(STUsesArray, NumSTUses, Inst); DEBUG({dbgs() << "Before asm: "; dumpStack();}); // With the stack layout fixed, rewrite the FP registers. @@ -1597,36 +1485,22 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { MachineOperand &Op = MI->getOperand(i); if (!Op.isReg() || Op.getReg() < X86::FP0 || Op.getReg() > X86::FP6) continue; + unsigned FPReg = getFPReg(Op); - Op.setReg(getSTReg(FPReg)); + + if (FRegIdx.count(i)) + // Operand with constraint "f". + Op.setReg(getSTReg(FPReg)); + else + // Operand with a single register class constraint ("t" or "u"). + Op.setReg(X86::ST0 + FPReg); } // Simulate the inline asm popping its inputs and pushing its outputs. StackTop -= NumSTPopped; - // Hold the fixed output registers in scratch FP registers. They will be - // transferred to real FP registers by copies. - NumPendingSTs = 0; - for (unsigned i = 0; i < NumSTDefs; ++i) { - unsigned SR = getScratchReg(); - pushReg(SR); - FPKills &= ~(1u << SR); - } for (unsigned i = 0; i < NumSTDefs; ++i) - PendingST[NumPendingSTs++] = getStackEntry(i); - DEBUG({dbgs() << "After asm: "; dumpStack();}); - - // If any of the ST defs were dead, pop them immediately. Our caller only - // handles dead FP defs. - MachineBasicBlock::iterator InsertPt = MI; - for (unsigned i = 0; STDefs & (1u << i); ++i) { - if (!(STDeadDefs & (1u << i))) - continue; - freeStackSlotAfter(InsertPt, PendingST[i]); - PendingST[i] = NumFPRegs; - } - while (NumPendingSTs && PendingST[NumPendingSTs - 1] == NumFPRegs) - --NumPendingSTs; + pushReg(NumSTDefs - i - 1); // If this asm kills any FP registers (is the last use of them) we must // explicitly emit pop instructions for them. Do this now after the asm has @@ -1636,11 +1510,12 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { // Note: this might be a non-optimal pop sequence. We might be able to do // better by trying to pop in stack order or something. while (FPKills) { - unsigned FPReg = CountTrailingZeros_32(FPKills); + unsigned FPReg = countTrailingZeros(FPKills); if (isLive(FPReg)) - freeStackSlotAfter(InsertPt, FPReg); + freeStackSlotAfter(Inst, FPReg); FPKills &= ~(1U << FPReg); } + // Don't delete the inline asm! return; } @@ -1653,14 +1528,15 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { Op.getReg() >= X86::FP0 && Op.getReg() <= X86::FP6); unsigned FPReg = getFPReg(Op); if (Op.isKill()) - moveToTop(FPReg, I); + moveToTop(FPReg, Inst); else - duplicateToTop(FPReg, FPReg, I); + duplicateToTop(FPReg, FPReg, Inst); // Emit the call. This will pop the operand. - BuildMI(*MBB, I, MI->getDebugLoc(), TII->get(X86::CALLpcrel32)) + BuildMI(*MBB, Inst, MI->getDebugLoc(), TII->get(X86::CALLpcrel32)) .addExternalSymbol("_ftol2") .addReg(X86::ST0, RegState::ImplicitKill) + .addReg(X86::ECX, RegState::ImplicitDefine) .addReg(X86::EAX, RegState::Define | RegState::Implicit) .addReg(X86::EDX, RegState::Define | RegState::Implicit) .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); @@ -1669,8 +1545,10 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { break; } - case X86::RET: - case X86::RETI: + case X86::RETQ: + case X86::RETL: + case X86::RETIL: + case X86::RETIQ: // If RET has an FP register use operand, pass the first one in ST(0) and // the second one in ST(1). @@ -1733,7 +1611,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { // Duplicate the TOS so that we return it twice. Just pick some other FPx // register to hold it. - unsigned NewReg = getScratchReg(); + unsigned NewReg = ScratchFPReg; duplicateToTop(FirstFPRegOp, NewReg, MI); FirstFPRegOp = NewReg; } @@ -1756,13 +1634,54 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { return; } - I = MBB->erase(I); // Remove the pseudo instruction + Inst = MBB->erase(Inst); // Remove the pseudo instruction // We want to leave I pointing to the previous instruction, but what if we // just erased the first instruction? - if (I == MBB->begin()) { + if (Inst == MBB->begin()) { DEBUG(dbgs() << "Inserting dummy KILL\n"); - I = BuildMI(*MBB, I, DebugLoc(), TII->get(TargetOpcode::KILL)); + Inst = BuildMI(*MBB, Inst, DebugLoc(), TII->get(TargetOpcode::KILL)); } else - --I; + --Inst; +} + +void FPS::setKillFlags(MachineBasicBlock &MBB) const { + const TargetRegisterInfo *TRI = + MBB.getParent()->getSubtarget().getRegisterInfo(); + LivePhysRegs LPR(TRI); + + LPR.addLiveOuts(&MBB); + + for (MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend(); + I != E; ++I) { + if (I->isDebugValue()) + continue; + + std::bitset<8> Defs; + SmallVector Uses; + MachineInstr &MI = *I; + + for (auto &MO : I->operands()) { + if (!MO.isReg()) + continue; + + unsigned Reg = MO.getReg() - X86::FP0; + + if (Reg >= 8) + continue; + + if (MO.isDef()) { + Defs.set(Reg); + if (!LPR.contains(MO.getReg())) + MO.setIsDead(); + } else + Uses.push_back(&MO); + } + + for (auto *MO : Uses) + if (Defs.test(getFPReg(*MO)) || !LPR.contains(MO->getReg())) + MO->setIsKill(); + + LPR.stepBackward(MI); + } }