X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=blobdiff_plain;f=lib%2FTarget%2FARM%2FARMConstantIslandPass.cpp;h=55c1684028c2add04535d69441885371e21177db;hp=b73acd184c6a366e9ee2e5ea441347f3b1ef52c9;hb=2921ff9ffcfd09db1c8d304188739d8d89de5611;hpb=59ecaae7b61ef0023ed1db4643937f7a57f61d10 diff --git a/lib/Target/ARM/ARMConstantIslandPass.cpp b/lib/Target/ARM/ARMConstantIslandPass.cpp index b73acd184c6..55c1684028c 100644 --- a/lib/Target/ARM/ARMConstantIslandPass.cpp +++ b/lib/Target/ARM/ARMConstantIslandPass.cpp @@ -13,29 +13,30 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "arm-cp-islands" #include "ARM.h" #include "ARMMachineFunctionInfo.h" -#include "ARMInstrInfo.h" -#include "Thumb2InstrInfo.h" #include "MCTargetDesc/ARMAddressingModes.h" +#include "Thumb2InstrInfo.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" -#include "llvm/Target/TargetData.h" -#include "llvm/Target/TargetMachine.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Support/CommandLine.h" +#include "llvm/Target/TargetMachine.h" #include using namespace llvm; +#define DEBUG_TYPE "arm-cp-islands" + STATISTIC(NumCPEs, "Number of constpool entries"); STATISTIC(NumSplit, "Number of uncond branches inserted"); STATISTIC(NumCBrFixed, "Number of cond branches fixed"); @@ -52,11 +53,6 @@ static cl::opt AdjustJumpTableBlocks("arm-adjust-jump-tables", cl::Hidden, cl::init(true), cl::desc("Adjust basic block layout to better use TB[BH]")); -// FIXME: This option should be removed once it has received sufficient testing. -static cl::opt -AlignConstantIslands("arm-align-constant-islands", cl::Hidden, cl::init(true), - cl::desc("Align constant islands in code")); - /// UnknownPadding - Return the worst case padding that could result from /// unknown offset bits. This does not include alignment padding caused by /// known offset bits. @@ -69,27 +65,6 @@ static inline unsigned UnknownPadding(unsigned LogAlign, unsigned KnownBits) { return 0; } -/// WorstCaseAlign - Assuming only the low KnownBits bits in Offset are exact, -/// add padding such that: -/// -/// 1. The result is aligned to 1 << LogAlign. -/// -/// 2. No other value of the unknown bits would require more padding. -/// -/// This may add more padding than is required to satisfy just one of the -/// constraints. It is necessary to compute alignment this way to guarantee -/// that we don't underestimate the padding before an aligned block. If the -/// real padding before a block is larger than we think, constant pool entries -/// may go out of range. -static inline unsigned WorstCaseAlign(unsigned Offset, unsigned LogAlign, - unsigned KnownBits) { - // Add the worst possible padding that the unknown bits could cause. - Offset += UnknownPadding(LogAlign, KnownBits); - - // Then align the result. - return RoundUpToAlignment(Offset, 1u << LogAlign); -} - namespace { /// ARMConstantIslands - Due to limited PC-relative displacements, ARM /// requires constant pool entries to be scattered among the instructions @@ -109,7 +84,12 @@ namespace { /// Offset - Distance from the beginning of the function to the beginning /// of this basic block. /// - /// The offset is always aligned as required by the basic block. + /// Offsets are computed assuming worst case padding before an aligned + /// block. This means that subtracting basic block offsets always gives a + /// conservative estimate of the real distance which may be smaller. + /// + /// Because worst case padding is used, the computed offset of an aligned + /// block may not actually be aligned. unsigned Offset; /// Size - Size of the basic block in bytes. If the block contains @@ -140,7 +120,12 @@ namespace { /// This number should be used to predict worst case padding when /// splitting the block. unsigned internalKnownBits() const { - return Unalign ? Unalign : KnownBits; + unsigned Bits = Unalign ? Unalign : KnownBits; + // If the block size isn't a multiple of the known bits, assume the + // worst case padding. + if (Size & ((1u << Bits) - 1)) + Bits = countTrailingZeros(Size); + return Bits; } /// Compute the offset immediately following this block. If LogAlign is @@ -152,7 +137,7 @@ namespace { if (!LA) return PO; // Add alignment padding from the terminator. - return WorstCaseAlign(PO, LA, internalKnownBits()); + return PO + UnknownPadding(LA, internalKnownBits()); } /// Compute the number of known low bits of postOffset. If this block @@ -198,11 +183,19 @@ namespace { unsigned MaxDisp; bool NegOk; bool IsSoImm; + bool KnownAlignment; CPUser(MachineInstr *mi, MachineInstr *cpemi, unsigned maxdisp, bool neg, bool soimm) - : MI(mi), CPEMI(cpemi), MaxDisp(maxdisp), NegOk(neg), IsSoImm(soimm) { + : MI(mi), CPEMI(cpemi), MaxDisp(maxdisp), NegOk(neg), IsSoImm(soimm), + KnownAlignment(false) { HighWaterMark = CPEMI->getParent(); } + /// getMaxDisp - Returns the maximum displacement supported by MI. + /// Correct for unknown alignment. + /// Conservatively subtract 2 bytes to handle weird alignment effects. + unsigned getMaxDisp() const { + return (KnownAlignment ? MaxDisp : MaxDisp - 2) - 2; + } }; /// CPUsers - Keep track of all of the machine instructions that use various @@ -221,12 +214,24 @@ namespace { }; /// CPEntries - Keep track of all of the constant pool entry machine - /// instructions. For each original constpool index (i.e. those that - /// existed upon entry to this pass), it keeps a vector of entries. - /// Original elements are cloned as we go along; the clones are - /// put in the vector of the original element, but have distinct CPIs. + /// instructions. For each original constpool index (i.e. those that existed + /// upon entry to this pass), it keeps a vector of entries. Original + /// elements are cloned as we go along; the clones are put in the vector of + /// the original element, but have distinct CPIs. + /// + /// The first half of CPEntries contains generic constants, the second half + /// contains jump tables. Use getCombinedIndex on a generic CPEMI to look up + /// which vector it will be in here. std::vector > CPEntries; + /// Maps a JT index to the offset in CPEntries containing copies of that + /// table. The equivalent map for a CONSTPOOL_ENTRY is the identity. + DenseMap JumpTableEntryIndices; + + /// Maps a JT index to the LEA that actually uses the index to calculate its + /// base address. + DenseMap JumpTableUserIndices; + /// ImmBranch - One per immediate branch, keeping the machine instruction /// pointer, conditional or unconditional, the max displacement, /// and (if isCond is true) the corresponding unconditional branch @@ -235,8 +240,8 @@ namespace { MachineInstr *MI; unsigned MaxDisp : 31; bool isCond : 1; - int UncondBr; - ImmBranch(MachineInstr *mi, unsigned maxdisp, bool cond, int ubr) + unsigned UncondBr; + ImmBranch(MachineInstr *mi, unsigned maxdisp, bool cond, unsigned ubr) : MI(mi), MaxDisp(maxdisp), isCond(cond), UncondBr(ubr) {} }; @@ -257,7 +262,7 @@ namespace { MachineFunction *MF; MachineConstantPool *MCP; - const ARMInstrInfo *TII; + const ARMBaseInstrInfo *TII; const ARMSubtarget *STI; ARMFunctionInfo *AFI; bool isThumb; @@ -267,57 +272,66 @@ namespace { static char ID; ARMConstantIslands() : MachineFunctionPass(ID) {} - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; - virtual const char *getPassName() const { + const char *getPassName() const override { return "ARM constant island placement and branch shortening pass"; } private: - void DoInitialPlacement(std::vector &CPEMIs); + void doInitialConstPlacement(std::vector &CPEMIs); + void doInitialJumpTablePlacement(std::vector &CPEMIs); + bool BBHasFallthrough(MachineBasicBlock *MBB); CPEntry *findConstPoolEntry(unsigned CPI, const MachineInstr *CPEMI); unsigned getCPELogAlign(const MachineInstr *CPEMI); - void JumpTableFunctionScan(); - void InitialFunctionScan(const std::vector &CPEMIs); - MachineBasicBlock *SplitBlockBeforeInstr(MachineInstr *MI); - void UpdateForInsertedWaterBlock(MachineBasicBlock *NewBB); - void AdjustBBOffsetsAfter(MachineBasicBlock *BB); - bool DecrementOldEntry(unsigned CPI, MachineInstr* CPEMI); - int LookForExistingCPEntry(CPUser& U, unsigned UserOffset); - bool LookForWater(CPUser&U, unsigned UserOffset, water_iterator &WaterIter); - void CreateNewWater(unsigned CPUserIndex, unsigned UserOffset, + void scanFunctionJumpTables(); + void initializeFunctionInfo(const std::vector &CPEMIs); + MachineBasicBlock *splitBlockBeforeInstr(MachineInstr *MI); + void updateForInsertedWaterBlock(MachineBasicBlock *NewBB); + void adjustBBOffsetsAfter(MachineBasicBlock *BB); + bool decrementCPEReferenceCount(unsigned CPI, MachineInstr* CPEMI); + unsigned getCombinedIndex(const MachineInstr *CPEMI); + int findInRangeCPEntry(CPUser& U, unsigned UserOffset); + bool findAvailableWater(CPUser&U, unsigned UserOffset, + water_iterator &WaterIter); + void createNewWater(unsigned CPUserIndex, unsigned UserOffset, MachineBasicBlock *&NewMBB); - bool HandleConstantPoolUser(unsigned CPUserIndex); - void RemoveDeadCPEMI(MachineInstr *CPEMI); - bool RemoveUnusedCPEntries(); - bool CPEIsInRange(MachineInstr *MI, unsigned UserOffset, - MachineInstr *CPEMI, unsigned Disp, bool NegOk, - bool DoDump = false); - bool WaterIsInRange(unsigned UserOffset, MachineBasicBlock *Water, + bool handleConstantPoolUser(unsigned CPUserIndex); + void removeDeadCPEMI(MachineInstr *CPEMI); + bool removeUnusedCPEntries(); + bool isCPEntryInRange(MachineInstr *MI, unsigned UserOffset, + MachineInstr *CPEMI, unsigned Disp, bool NegOk, + bool DoDump = false); + bool isWaterInRange(unsigned UserOffset, MachineBasicBlock *Water, CPUser &U, unsigned &Growth); - bool BBIsInRange(MachineInstr *MI, MachineBasicBlock *BB, unsigned Disp); - bool FixUpImmediateBr(ImmBranch &Br); - bool FixUpConditionalBr(ImmBranch &Br); - bool FixUpUnconditionalBr(ImmBranch &Br); - bool UndoLRSpillRestore(); - bool OptimizeThumb2Instructions(); - bool OptimizeThumb2Branches(); - bool ReorderThumb2JumpTables(); - bool OptimizeThumb2JumpTables(); - MachineBasicBlock *AdjustJTTargetBlockForward(MachineBasicBlock *BB, + bool isBBInRange(MachineInstr *MI, MachineBasicBlock *BB, unsigned Disp); + bool fixupImmediateBr(ImmBranch &Br); + bool fixupConditionalBr(ImmBranch &Br); + bool fixupUnconditionalBr(ImmBranch &Br); + bool undoLRSpillRestore(); + bool mayOptimizeThumb2Instruction(const MachineInstr *MI) const; + bool optimizeThumb2Instructions(); + bool optimizeThumb2Branches(); + bool reorderThumb2JumpTables(); + bool preserveBaseRegister(MachineInstr *JumpMI, MachineInstr *LEAMI, + unsigned &DeadSize, bool &CanDeleteLEA, + bool &BaseRegKill); + bool optimizeThumb2JumpTables(); + MachineBasicBlock *adjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB); - void ComputeBlockSize(MachineBasicBlock *MBB); - unsigned GetOffsetOf(MachineInstr *MI) const; + void computeBlockSize(MachineBasicBlock *MBB); + unsigned getOffsetOf(MachineInstr *MI) const; + unsigned getUserOffset(CPUser&) const; void dumpBBs(); void verify(); - bool OffsetIsInRange(unsigned UserOffset, unsigned TrialOffset, + bool isOffsetInRange(unsigned UserOffset, unsigned TrialOffset, unsigned Disp, bool NegativeOK, bool IsSoImm = false); - bool OffsetIsInRange(unsigned UserOffset, unsigned TrialOffset, + bool isOffsetInRange(unsigned UserOffset, unsigned TrialOffset, const CPUser &U) { - return OffsetIsInRange(UserOffset, TrialOffset, - U.MaxDisp, U.NegOk, U.IsSoImm); + return isOffsetInRange(UserOffset, TrialOffset, + U.getMaxDisp(), U.NegOk, U.IsSoImm); } }; char ARMConstantIslands::ID = 0; @@ -326,21 +340,27 @@ namespace { /// verify - check BBOffsets, BBSizes, alignment of islands void ARMConstantIslands::verify() { #ifndef NDEBUG - for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end(); - MBBI != E; ++MBBI) { - MachineBasicBlock *MBB = MBBI; - unsigned Align = MBB->getAlignment(); - unsigned MBBId = MBB->getNumber(); - assert(BBInfo[MBBId].Offset % (1u << Align) == 0); - assert(!MBBId || BBInfo[MBBId - 1].postOffset() <= BBInfo[MBBId].Offset); - } + assert(std::is_sorted(MF->begin(), MF->end(), + [this](const MachineBasicBlock &LHS, + const MachineBasicBlock &RHS) { + return BBInfo[LHS.getNumber()].postOffset() < + BBInfo[RHS.getNumber()].postOffset(); + })); + DEBUG(dbgs() << "Verifying " << CPUsers.size() << " CP users.\n"); for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) { CPUser &U = CPUsers[i]; - unsigned UserOffset = GetOffsetOf(U.MI) + (isThumb ? 4 : 8); - unsigned CPEOffset = GetOffsetOf(U.CPEMI); - unsigned Disp = UserOffset < CPEOffset ? CPEOffset - UserOffset : - UserOffset - CPEOffset; - assert(Disp <= U.MaxDisp || "Constant pool entry out of range!"); + unsigned UserOffset = getUserOffset(U); + // Verify offset using the real max displacement without the safety + // adjustment. + if (isCPEntryInRange(U.MI, UserOffset, U.CPEMI, U.getMaxDisp()+2, U.NegOk, + /* DoDump = */ true)) { + DEBUG(dbgs() << "OK\n"); + continue; + } + DEBUG(dbgs() << "Out of range.\n"); + dumpBBs(); + DEBUG(MF->dump()); + llvm_unreachable("Constant pool entry out of range!"); } #endif } @@ -373,9 +393,9 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { << MCP->getConstants().size() << " CP entries, aligned to " << MCP->getConstantPoolAlignment() << " bytes *****\n"); - TII = (const ARMInstrInfo*)MF->getTarget().getInstrInfo(); + STI = &static_cast(MF->getSubtarget()); + TII = STI->getInstrInfo(); AFI = MF->getInfo(); - STI = &MF->getTarget().getSubtarget(); isThumb = AFI->isThumbFunction(); isThumb1 = AFI->isThumb1OnlyFunction(); @@ -383,6 +403,9 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { HasFarJump = false; + // This pass invalidates liveness information when it splits basic blocks. + MF->getRegInfo().invalidateLiveness(); + // Renumber all of the machine basic blocks in the function, guaranteeing that // the numbers agree with the position of the block in the function. MF->RenumberBlocks(); @@ -391,26 +414,22 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { // of the TB[BH] instructions. bool MadeChange = false; if (isThumb2 && AdjustJumpTableBlocks) { - JumpTableFunctionScan(); - MadeChange |= ReorderThumb2JumpTables(); + scanFunctionJumpTables(); + MadeChange |= reorderThumb2JumpTables(); // Data is out of date, so clear it. It'll be re-computed later. T2JumpTables.clear(); // Blocks may have shifted around. Keep the numbering up to date. MF->RenumberBlocks(); } - // Thumb1 functions containing constant pools get 4-byte alignment. - // This is so we can keep exact track of where the alignment padding goes. - - // ARM and Thumb2 functions need to be 4-byte aligned. - if (!isThumb1) - MF->EnsureAlignment(2); // 2 = log2(4) - // Perform the initial placement of the constant pool entries. To start with, // we put them all at the end of the function. std::vector CPEMIs; if (!MCP->isEmpty()) - DoInitialPlacement(CPEMIs); + doInitialConstPlacement(CPEMIs); + + if (MF->getJumpTableInfo()) + doInitialJumpTablePlacement(CPEMIs); /// The next UID to take is the first unused one. AFI->initPICLabelUId(CPEMIs.size()); @@ -418,13 +437,17 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { // Do the initial scan of the function, building up information about the // sizes of each block, the location of all the water, and finding all of the // constant pool users. - InitialFunctionScan(CPEMIs); + initializeFunctionInfo(CPEMIs); CPEMIs.clear(); DEBUG(dumpBBs()); + // Functions with jump tables need an alignment of 4 because they use the ADR + // instruction, which aligns the PC to 4 bytes before adding an offset. + if (!T2JumpTables.empty()) + MF->ensureAlignment(2); /// Remove dead constant pool entries. - MadeChange |= RemoveUnusedCPEntries(); + MadeChange |= removeUnusedCPEntries(); // Iteratively place constant pool entries and fix up branches until there // is no change. @@ -433,9 +456,9 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { DEBUG(dbgs() << "Beginning CP iteration #" << NoCPIters << '\n'); bool CPChange = false; for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) - CPChange |= HandleConstantPoolUser(i); + CPChange |= handleConstantPoolUser(i); if (CPChange && ++NoCPIters > 30) - llvm_unreachable("Constant Island pass failed to converge!"); + report_fatal_error("Constant Island pass failed to converge!"); DEBUG(dumpBBs()); // Clear NewWaterList now. If we split a block for branches, it should @@ -445,9 +468,9 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { DEBUG(dbgs() << "Beginning BR iteration #" << NoBRIters << '\n'); bool BRChange = false; for (unsigned i = 0, e = ImmBranches.size(); i != e; ++i) - BRChange |= FixUpImmediateBr(ImmBranches[i]); + BRChange |= fixupImmediateBr(ImmBranches[i]); if (BRChange && ++NoBRIters > 30) - llvm_unreachable("Branch Fix Up pass failed to converge!"); + report_fatal_error("Branch Fix Up pass failed to converge!"); DEBUG(dumpBBs()); if (!CPChange && !BRChange) @@ -457,7 +480,7 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { // Shrink 32-bit Thumb2 branch, load, and store instructions. if (isThumb2 && !STI->prefers32BitThumb()) - MadeChange |= OptimizeThumb2Instructions(); + MadeChange |= optimizeThumb2Instructions(); // After a while, this might be made debug-only, but it is not expensive. verify(); @@ -465,13 +488,14 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { // If LR has been forced spilled and no far jump (i.e. BL) has been issued, // undo the spill / restore of LR if possible. if (isThumb && !HasFarJump && AFI->isLRSpilledForFarJump()) - MadeChange |= UndoLRSpillRestore(); + MadeChange |= undoLRSpillRestore(); // Save the mapping between original and cloned constpool entries. for (unsigned i = 0, e = CPEntries.size(); i != e; ++i) { for (unsigned j = 0, je = CPEntries[i].size(); j != je; ++j) { const CPEntry & CPE = CPEntries[i][j]; - AFI->recordCPEClone(i, CPE.CPI); + if (CPE.CPEMI && CPE.CPEMI->getOperand(1).isCPI()) + AFI->recordCPEClone(i, CPE.CPI); } } @@ -481,6 +505,8 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { WaterList.clear(); CPUsers.clear(); CPEntries.clear(); + JumpTableEntryIndices.clear(); + JumpTableUserIndices.clear(); ImmBranches.clear(); PushPopMIs.clear(); T2JumpTables.clear(); @@ -488,10 +514,10 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) { return MadeChange; } -/// DoInitialPlacement - Perform the initial placement of the constant pool -/// entries. To start with, we put them all at the end of the function. +/// \brief Perform the initial placement of the regular constant pool entries. +/// To start with, we put them all at the end of the function. void -ARMConstantIslands::DoInitialPlacement(std::vector &CPEMIs) { +ARMConstantIslands::doInitialConstPlacement(std::vector &CPEMIs) { // Create the basic block to hold the CPE's. MachineBasicBlock *BB = MF->CreateMachineBasicBlock(); MF->push_back(BB); @@ -500,12 +526,11 @@ ARMConstantIslands::DoInitialPlacement(std::vector &CPEMIs) { unsigned MaxAlign = Log2_32(MCP->getConstantPoolAlignment()); // Mark the basic block as required by the const-pool. - // If AlignConstantIslands isn't set, use 4-byte alignment for everything. - BB->setAlignment(AlignConstantIslands ? MaxAlign : 2); + BB->setAlignment(MaxAlign); // The function needs to be as aligned as the basic blocks. The linker may // move functions around based on their alignment. - MF->EnsureAlignment(BB->getAlignment()); + MF->ensureAlignment(BB->getAlignment()); // Order the entries in BB by descending alignment. That ensures correct // alignment of all entries as long as BB is sufficiently aligned. Keep @@ -517,7 +542,7 @@ ARMConstantIslands::DoInitialPlacement(std::vector &CPEMIs) { // identity mapping of CPI's to CPE's. const std::vector &CPs = MCP->getConstants(); - const TargetData &TD = *MF->getTarget().getTargetData(); + const DataLayout &TD = MF->getDataLayout(); for (unsigned i = 0, e = CPs.size(); i != e; ++i) { unsigned Size = TD.getTypeAllocSize(CPs[i].getType()); assert(Size >= 4 && "Too small constant pool entry"); @@ -542,31 +567,95 @@ ARMConstantIslands::DoInitialPlacement(std::vector &CPEMIs) { InsPoint[a] = CPEMI; // Add a new CPEntry, but no corresponding CPUser yet. - std::vector CPEs; - CPEs.push_back(CPEntry(CPEMI, i)); - CPEntries.push_back(CPEs); + CPEntries.emplace_back(1, CPEntry(CPEMI, i)); ++NumCPEs; - DEBUG(dbgs() << "Moved CPI#" << i << " to end of function\n"); + DEBUG(dbgs() << "Moved CPI#" << i << " to end of function, size = " + << Size << ", align = " << Align <<'\n'); } DEBUG(BB->dump()); } +/// \brief Do initial placement of the jump tables. Because Thumb2's TBB and TBH +/// instructions can be made more efficient if the jump table immediately +/// follows the instruction, it's best to place them immediately next to their +/// jumps to begin with. In almost all cases they'll never be moved from that +/// position. +void ARMConstantIslands::doInitialJumpTablePlacement( + std::vector &CPEMIs) { + unsigned i = CPEntries.size(); + auto MJTI = MF->getJumpTableInfo(); + const std::vector &JT = MJTI->getJumpTables(); + + MachineBasicBlock *LastCorrectlyNumberedBB = nullptr; + for (MachineBasicBlock &MBB : *MF) { + auto MI = MBB.getLastNonDebugInstr(); + if (MI == MBB.end()) + continue; + + unsigned JTOpcode; + switch (MI->getOpcode()) { + default: + continue; + case ARM::BR_JTadd: + case ARM::BR_JTr: + case ARM::tBR_JTr: + case ARM::BR_JTm: + JTOpcode = ARM::JUMPTABLE_ADDRS; + break; + case ARM::t2BR_JT: + JTOpcode = ARM::JUMPTABLE_INSTS; + break; + case ARM::t2TBB_JT: + JTOpcode = ARM::JUMPTABLE_TBB; + break; + case ARM::t2TBH_JT: + JTOpcode = ARM::JUMPTABLE_TBH; + break; + } + + unsigned NumOps = MI->getDesc().getNumOperands(); + MachineOperand JTOp = + MI->getOperand(NumOps - (MI->isPredicable() ? 2 : 1)); + unsigned JTI = JTOp.getIndex(); + unsigned Size = JT[JTI].MBBs.size() * sizeof(uint32_t); + MachineBasicBlock *JumpTableBB = MF->CreateMachineBasicBlock(); + MF->insert(std::next(MachineFunction::iterator(MBB)), JumpTableBB); + MachineInstr *CPEMI = BuildMI(*JumpTableBB, JumpTableBB->begin(), + DebugLoc(), TII->get(JTOpcode)) + .addImm(i++) + .addJumpTableIndex(JTI) + .addImm(Size); + CPEMIs.push_back(CPEMI); + CPEntries.emplace_back(1, CPEntry(CPEMI, JTI)); + JumpTableEntryIndices.insert(std::make_pair(JTI, CPEntries.size() - 1)); + if (!LastCorrectlyNumberedBB) + LastCorrectlyNumberedBB = &MBB; + } + + // If we did anything then we need to renumber the subsequent blocks. + if (LastCorrectlyNumberedBB) + MF->RenumberBlocks(LastCorrectlyNumberedBB); +} + /// BBHasFallthrough - Return true if the specified basic block can fallthrough /// into the block immediately after it. -static bool BBHasFallthrough(MachineBasicBlock *MBB) { +bool ARMConstantIslands::BBHasFallthrough(MachineBasicBlock *MBB) { // Get the next machine basic block in the function. - MachineFunction::iterator MBBI = MBB; + MachineFunction::iterator MBBI = MBB->getIterator(); // Can't fall off end of function. - if (llvm::next(MBBI) == MBB->getParent()->end()) + if (std::next(MBBI) == MBB->getParent()->end()) return false; - MachineBasicBlock *NextBB = llvm::next(MBBI); - for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(), - E = MBB->succ_end(); I != E; ++I) - if (*I == NextBB) - return true; + MachineBasicBlock *NextBB = &*std::next(MBBI); + if (std::find(MBB->succ_begin(), MBB->succ_end(), NextBB) == MBB->succ_end()) + return false; - return false; + // Try to analyze the end of the block. A potential fallthrough may already + // have an unconditional branch for whatever reason. + MachineBasicBlock *TBB, *FBB; + SmallVector Cond; + bool TooDifficult = TII->AnalyzeBranch(*MBB, TBB, FBB, Cond); + return TooDifficult || FBB == nullptr; } /// findConstPoolEntry - Given the constpool index and CONSTPOOL_ENTRY MI, @@ -581,29 +670,37 @@ ARMConstantIslands::CPEntry if (CPEs[i].CPEMI == CPEMI) return &CPEs[i]; } - return NULL; + return nullptr; } /// getCPELogAlign - Returns the required alignment of the constant pool entry /// represented by CPEMI. Alignment is measured in log2(bytes) units. unsigned ARMConstantIslands::getCPELogAlign(const MachineInstr *CPEMI) { - assert(CPEMI && CPEMI->getOpcode() == ARM::CONSTPOOL_ENTRY); - - // Everything is 4-byte aligned unless AlignConstantIslands is set. - if (!AlignConstantIslands) + switch (CPEMI->getOpcode()) { + case ARM::CONSTPOOL_ENTRY: + break; + case ARM::JUMPTABLE_TBB: + return 0; + case ARM::JUMPTABLE_TBH: + case ARM::JUMPTABLE_INSTS: + return 1; + case ARM::JUMPTABLE_ADDRS: return 2; + default: + llvm_unreachable("unknown constpool entry kind"); + } - unsigned CPI = CPEMI->getOperand(1).getIndex(); + unsigned CPI = getCombinedIndex(CPEMI); assert(CPI < MCP->getConstants().size() && "Invalid constant pool index."); unsigned Align = MCP->getConstants()[CPI].getAlignment(); assert(isPowerOf2_32(Align) && "Invalid CPE alignment"); return Log2_32(Align); } -/// JumpTableFunctionScan - Do a scan of the function, building up +/// scanFunctionJumpTables - Do a scan of the function, building up /// information about the sizes of each block and the locations of all /// the jump tables. -void ARMConstantIslands::JumpTableFunctionScan() { +void ARMConstantIslands::scanFunctionJumpTables() { for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end(); MBBI != E; ++MBBI) { MachineBasicBlock &MBB = *MBBI; @@ -615,11 +712,11 @@ void ARMConstantIslands::JumpTableFunctionScan() { } } -/// InitialFunctionScan - Do the initial scan of the function, building up +/// initializeFunctionInfo - Do the initial scan of the function, building up /// information about the sizes of each block, the location of all the water, /// and finding all of the constant pool users. void ARMConstantIslands:: -InitialFunctionScan(const std::vector &CPEMIs) { +initializeFunctionInfo(const std::vector &CPEMIs) { BBInfo.clear(); BBInfo.resize(MF->getNumBlockIDs()); @@ -627,15 +724,15 @@ InitialFunctionScan(const std::vector &CPEMIs) { // has any inline assembly in it. If so, we have to be conservative about // alignment assumptions, as we don't know for sure the size of any // instructions in the inline assembly. - for (MachineFunction::iterator I = MF->begin(), E = MF->end(); I != E; ++I) - ComputeBlockSize(I); + for (MachineBasicBlock &MBB : *MF) + computeBlockSize(&MBB); // The known bits of the entry block offset are determined by the function // alignment. BBInfo.front().KnownBits = MF->getAlignment(); // Compute block offsets and known bits. - AdjustBBOffsetsAfter(MF->begin()); + adjustBBOffsetsAfter(&MF->front()); // Now go back through the instructions and build up our data structures. for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end(); @@ -652,7 +749,7 @@ InitialFunctionScan(const std::vector &CPEMIs) { if (I->isDebugValue()) continue; - int Opc = I->getOpcode(); + unsigned Opc = I->getOpcode(); if (I->isBranch()) { bool isCond = false; unsigned Bits = 0; @@ -702,12 +799,14 @@ InitialFunctionScan(const std::vector &CPEMIs) { if (Opc == ARM::tPUSH || Opc == ARM::tPOP_RET) PushPopMIs.push_back(I); - if (Opc == ARM::CONSTPOOL_ENTRY) + if (Opc == ARM::CONSTPOOL_ENTRY || Opc == ARM::JUMPTABLE_ADDRS || + Opc == ARM::JUMPTABLE_INSTS || Opc == ARM::JUMPTABLE_TBB || + Opc == ARM::JUMPTABLE_TBH) continue; // Scan the instructions for constant pool operands. for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op) - if (I->getOperand(op).isCPI()) { + if (I->getOperand(op).isCPI() || I->getOperand(op).isJTI()) { // We found one. The addressing mode tells us the max displacement // from the PC that this instruction permits. @@ -720,10 +819,10 @@ InitialFunctionScan(const std::vector &CPEMIs) { switch (Opc) { default: llvm_unreachable("Unknown addressing mode for CP reference!"); - break; // Taking the address of a CP entry. case ARM::LEApcrel: + case ARM::LEApcrelJT: // This takes a SoImm, which is 8 bit immediate rotated. We'll // pretend the maximum offset is 255 * 4. Since each instruction // 4 byte wide, this is always correct. We'll check for other @@ -734,14 +833,17 @@ InitialFunctionScan(const std::vector &CPEMIs) { IsSoImm = true; break; case ARM::t2LEApcrel: + case ARM::t2LEApcrelJT: Bits = 12; NegOk = true; break; case ARM::tLEApcrel: + case ARM::tLEApcrelJT: Bits = 8; Scale = 4; break; + case ARM::LDRBi12: case ARM::LDRi12: case ARM::LDRcp: case ARM::t2LDRpci: @@ -764,6 +866,11 @@ InitialFunctionScan(const std::vector &CPEMIs) { // Remember that this is a user of a CP entry. unsigned CPI = I->getOperand(op).getIndex(); + if (I->getOperand(op).isJTI()) { + JumpTableUserIndices.insert(std::make_pair(CPI, CPUsers.size())); + CPI = JumpTableEntryIndices[CPI]; + } + MachineInstr *CPEMI = CPEMIs[CPI]; unsigned MaxOffs = ((1 << Bits)-1) * Scale; CPUsers.push_back(CPUser(I, CPEMI, MaxOffs, NegOk, IsSoImm)); @@ -781,9 +888,9 @@ InitialFunctionScan(const std::vector &CPEMIs) { } } -/// ComputeBlockSize - Compute the size and some alignment information for MBB. +/// computeBlockSize - Compute the size and some alignment information for MBB. /// This function updates BBInfo directly. -void ARMConstantIslands::ComputeBlockSize(MachineBasicBlock *MBB) { +void ARMConstantIslands::computeBlockSize(MachineBasicBlock *MBB) { BasicBlockInfo &BBI = BBInfo[MBB->getNumber()]; BBI.Size = 0; BBI.Unalign = 0; @@ -796,19 +903,22 @@ void ARMConstantIslands::ComputeBlockSize(MachineBasicBlock *MBB) { // The actual size may be smaller, but still a multiple of the instr size. if (I->isInlineAsm()) BBI.Unalign = isThumb ? 1 : 2; + // Also consider instructions that may be shrunk later. + else if (isThumb && mayOptimizeThumb2Instruction(I)) + BBI.Unalign = 1; } // tBR_JTr contains a .align 2 directive. if (!MBB->empty() && MBB->back().getOpcode() == ARM::tBR_JTr) { BBI.PostAlign = 2; - MBB->getParent()->EnsureAlignment(2); + MBB->getParent()->ensureAlignment(2); } } -/// GetOffsetOf - Return the current offset of the specified machine instruction +/// getOffsetOf - Return the current offset of the specified machine instruction /// from the start of the function. This offset changes as stuff is moved /// around inside the function. -unsigned ARMConstantIslands::GetOffsetOf(MachineInstr *MI) const { +unsigned ARMConstantIslands::getOffsetOf(MachineInstr *MI) const { MachineBasicBlock *MBB = MI->getParent(); // The offset is composed of two things: the sum of the sizes of all MBB's @@ -817,11 +927,11 @@ unsigned ARMConstantIslands::GetOffsetOf(MachineInstr *MI) const { unsigned Offset = BBInfo[MBB->getNumber()].Offset; // Sum instructions before MI in MBB. - for (MachineBasicBlock::iterator I = MBB->begin(); ; ++I) { + for (MachineBasicBlock::iterator I = MBB->begin(); &*I != MI; ++I) { assert(I != MBB->end() && "Didn't find MI in its own basic block?"); - if (&*I == MI) return Offset; Offset += TII->GetInstSizeInBytes(I); } + return Offset; } /// CompareMBBNumbers - Little predicate function to sort the WaterList by MBB @@ -831,10 +941,10 @@ static bool CompareMBBNumbers(const MachineBasicBlock *LHS, return LHS->getNumber() < RHS->getNumber(); } -/// UpdateForInsertedWaterBlock - When a block is newly inserted into the +/// updateForInsertedWaterBlock - When a block is newly inserted into the /// machine function, it upsets all of the block numbers. Renumber the blocks /// and update the arrays that parallel this numbering. -void ARMConstantIslands::UpdateForInsertedWaterBlock(MachineBasicBlock *NewBB) { +void ARMConstantIslands::updateForInsertedWaterBlock(MachineBasicBlock *NewBB) { // Renumber the MBB's to keep them consecutive. NewBB->getParent()->RenumberBlocks(NewBB); @@ -854,13 +964,13 @@ void ARMConstantIslands::UpdateForInsertedWaterBlock(MachineBasicBlock *NewBB) { /// Split the basic block containing MI into two blocks, which are joined by /// an unconditional branch. Update data structures and renumber blocks to /// account for this change and returns the newly created block. -MachineBasicBlock *ARMConstantIslands::SplitBlockBeforeInstr(MachineInstr *MI) { +MachineBasicBlock *ARMConstantIslands::splitBlockBeforeInstr(MachineInstr *MI) { MachineBasicBlock *OrigBB = MI->getParent(); // Create a new MBB for the code after the OrigBB. MachineBasicBlock *NewBB = MF->CreateMachineBasicBlock(OrigBB->getBasicBlock()); - MachineFunction::iterator MBBI = OrigBB; ++MBBI; + MachineFunction::iterator MBBI = ++OrigBB->getIterator(); MF->insert(MBBI, NewBB); // Splice the instructions starting with MI over to NewBB. @@ -885,7 +995,7 @@ MachineBasicBlock *ARMConstantIslands::SplitBlockBeforeInstr(MachineInstr *MI) { OrigBB->addSuccessor(NewBB); // Update internal data structures to account for the newly inserted MBB. - // This is almost the same as UpdateForInsertedWaterBlock, except that + // This is almost the same as updateForInsertedWaterBlock, except that // the Water goes after OrigBB, not NewBB. MF->RenumberBlocks(NewBB); @@ -902,7 +1012,7 @@ MachineBasicBlock *ARMConstantIslands::SplitBlockBeforeInstr(MachineInstr *MI) { CompareMBBNumbers); MachineBasicBlock* WaterBB = *IP; if (WaterBB == OrigBB) - WaterList.insert(llvm::next(IP), NewBB); + WaterList.insert(std::next(IP), NewBB); else WaterList.insert(IP, OrigBB); NewWaterList.insert(OrigBB); @@ -912,31 +1022,51 @@ MachineBasicBlock *ARMConstantIslands::SplitBlockBeforeInstr(MachineInstr *MI) { // the new jump we added. (It should be possible to do this without // recounting everything, but it's very confusing, and this is rarely // executed.) - ComputeBlockSize(OrigBB); + computeBlockSize(OrigBB); // Figure out how large the NewMBB is. As the second half of the original // block, it may contain a tablejump. - ComputeBlockSize(NewBB); + computeBlockSize(NewBB); // All BBOffsets following these blocks must be modified. - AdjustBBOffsetsAfter(OrigBB); + adjustBBOffsetsAfter(OrigBB); return NewBB; } -/// OffsetIsInRange - Checks whether UserOffset (the location of a constant pool +/// getUserOffset - Compute the offset of U.MI as seen by the hardware +/// displacement computation. Update U.KnownAlignment to match its current +/// basic block location. +unsigned ARMConstantIslands::getUserOffset(CPUser &U) const { + unsigned UserOffset = getOffsetOf(U.MI); + const BasicBlockInfo &BBI = BBInfo[U.MI->getParent()->getNumber()]; + unsigned KnownBits = BBI.internalKnownBits(); + + // The value read from PC is offset from the actual instruction address. + UserOffset += (isThumb ? 4 : 8); + + // Because of inline assembly, we may not know the alignment (mod 4) of U.MI. + // Make sure U.getMaxDisp() returns a constrained range. + U.KnownAlignment = (KnownBits >= 2); + + // On Thumb, offsets==2 mod 4 are rounded down by the hardware for + // purposes of the displacement computation; compensate for that here. + // For unknown alignments, getMaxDisp() constrains the range instead. + if (isThumb && U.KnownAlignment) + UserOffset &= ~3u; + + return UserOffset; +} + +/// isOffsetInRange - Checks whether UserOffset (the location of a constant pool /// reference) is within MaxDisp of TrialOffset (a proposed location of a /// constant pool entry). -bool ARMConstantIslands::OffsetIsInRange(unsigned UserOffset, +/// UserOffset is computed by getUserOffset above to include PC adjustments. If +/// the mod 4 alignment of UserOffset is not known, the uncertainty must be +/// subtracted from MaxDisp instead. CPUser::getMaxDisp() does that. +bool ARMConstantIslands::isOffsetInRange(unsigned UserOffset, unsigned TrialOffset, unsigned MaxDisp, bool NegativeOK, bool IsSoImm) { - // On Thumb offsets==2 mod 4 are rounded down by the hardware for - // purposes of the displacement computation; compensate for that here. - // Effectively, the valid range of displacements is 2 bytes smaller for such - // references. - if (isThumb && UserOffset%4 !=0) - UserOffset -= 2; - if (UserOffset <= TrialOffset) { // User before the Trial. if (TrialOffset - UserOffset <= MaxDisp) @@ -950,17 +1080,17 @@ bool ARMConstantIslands::OffsetIsInRange(unsigned UserOffset, return false; } -/// WaterIsInRange - Returns true if a CPE placed after the specified +/// isWaterInRange - Returns true if a CPE placed after the specified /// Water (a basic block) will be in range for the specific MI. /// /// Compute how much the function will grow by inserting a CPE after Water. -bool ARMConstantIslands::WaterIsInRange(unsigned UserOffset, +bool ARMConstantIslands::isWaterInRange(unsigned UserOffset, MachineBasicBlock* Water, CPUser &U, unsigned &Growth) { unsigned CPELogAlign = getCPELogAlign(U.CPEMI); unsigned CPEOffset = BBInfo[Water->getNumber()].postOffset(CPELogAlign); unsigned NextBlockOffset, NextBlockAlignment; - MachineFunction::const_iterator NextBlock = Water; + MachineFunction::const_iterator NextBlock = Water->getIterator(); if (++NextBlock == MF->end()) { NextBlockOffset = BBInfo[Water->getNumber()].postOffset(); NextBlockAlignment = 0; @@ -981,7 +1111,7 @@ bool ARMConstantIslands::WaterIsInRange(unsigned UserOffset, Growth += OffsetToAlignment(CPEEnd, 1u << NextBlockAlignment); // If the CPE is to be inserted before the instruction, that will raise - // the offset of the instruction. Also account for unknown alignment padding + // the offset of the instruction. Also account for unknown alignment padding // in blocks between CPE and the user. if (CPEOffset < UserOffset) UserOffset += Growth + UnknownPadding(MF->getAlignment(), CPELogAlign); @@ -989,16 +1119,15 @@ bool ARMConstantIslands::WaterIsInRange(unsigned UserOffset, // CPE fits in existing padding. Growth = 0; - return OffsetIsInRange(UserOffset, CPEOffset, U); + return isOffsetInRange(UserOffset, CPEOffset, U); } -/// CPEIsInRange - Returns true if the distance between specific MI and +/// isCPEntryInRange - Returns true if the distance between specific MI and /// specific ConstPool entry instruction can fit in MI's displacement field. -bool ARMConstantIslands::CPEIsInRange(MachineInstr *MI, unsigned UserOffset, +bool ARMConstantIslands::isCPEntryInRange(MachineInstr *MI, unsigned UserOffset, MachineInstr *CPEMI, unsigned MaxDisp, bool NegOk, bool DoDump) { - unsigned CPEOffset = GetOffsetOf(CPEMI); - assert(CPEOffset % 4 == 0 && "Misaligned CPE"); + unsigned CPEOffset = getOffsetOf(CPEMI); if (DoDump) { DEBUG({ @@ -1014,7 +1143,7 @@ bool ARMConstantIslands::CPEIsInRange(MachineInstr *MI, unsigned UserOffset, }); } - return OffsetIsInRange(UserOffset, CPEOffset, MaxDisp, NegOk); + return isOffsetInRange(UserOffset, CPEOffset, MaxDisp, NegOk); } #ifndef NDEBUG @@ -1034,7 +1163,7 @@ static bool BBIsJumpedOver(MachineBasicBlock *MBB) { } #endif // NDEBUG -void ARMConstantIslands::AdjustBBOffsetsAfter(MachineBasicBlock *BB) { +void ARMConstantIslands::adjustBBOffsetsAfter(MachineBasicBlock *BB) { unsigned BBNum = BB->getNumber(); for(unsigned i = BBNum + 1, e = MF->getNumBlockIDs(); i < e; ++i) { // Get the offset and known bits at the end of the layout predecessor. @@ -1056,52 +1185,62 @@ void ARMConstantIslands::AdjustBBOffsetsAfter(MachineBasicBlock *BB) { } } -/// DecrementOldEntry - find the constant pool entry with index CPI +/// decrementCPEReferenceCount - find the constant pool entry with index CPI /// and instruction CPEMI, and decrement its refcount. If the refcount /// becomes 0 remove the entry and instruction. Returns true if we removed /// the entry, false if we didn't. -bool ARMConstantIslands::DecrementOldEntry(unsigned CPI, MachineInstr *CPEMI) { +bool ARMConstantIslands::decrementCPEReferenceCount(unsigned CPI, + MachineInstr *CPEMI) { // Find the old entry. Eliminate it if it is no longer used. CPEntry *CPE = findConstPoolEntry(CPI, CPEMI); assert(CPE && "Unexpected!"); if (--CPE->RefCount == 0) { - RemoveDeadCPEMI(CPEMI); - CPE->CPEMI = NULL; + removeDeadCPEMI(CPEMI); + CPE->CPEMI = nullptr; --NumCPEs; return true; } return false; } +unsigned ARMConstantIslands::getCombinedIndex(const MachineInstr *CPEMI) { + if (CPEMI->getOperand(1).isCPI()) + return CPEMI->getOperand(1).getIndex(); + + return JumpTableEntryIndices[CPEMI->getOperand(1).getIndex()]; +} + /// LookForCPEntryInRange - see if the currently referenced CPE is in range; /// if not, see if an in-range clone of the CPE is in range, and if so, /// change the data structures so the user references the clone. Returns: /// 0 = no existing entry found /// 1 = entry found, and there were no code insertions or deletions /// 2 = entry found, and there were code insertions or deletions -int ARMConstantIslands::LookForExistingCPEntry(CPUser& U, unsigned UserOffset) +int ARMConstantIslands::findInRangeCPEntry(CPUser& U, unsigned UserOffset) { MachineInstr *UserMI = U.MI; MachineInstr *CPEMI = U.CPEMI; // Check to see if the CPE is already in-range. - if (CPEIsInRange(UserMI, UserOffset, CPEMI, U.MaxDisp, U.NegOk, true)) { + if (isCPEntryInRange(UserMI, UserOffset, CPEMI, U.getMaxDisp(), U.NegOk, + true)) { DEBUG(dbgs() << "In range\n"); return 1; } // No. Look for previously created clones of the CPE that are in range. - unsigned CPI = CPEMI->getOperand(1).getIndex(); + unsigned CPI = getCombinedIndex(CPEMI); std::vector &CPEs = CPEntries[CPI]; for (unsigned i = 0, e = CPEs.size(); i != e; ++i) { // We already tried this one if (CPEs[i].CPEMI == CPEMI) continue; // Removing CPEs can leave empty entries, skip - if (CPEs[i].CPEMI == NULL) + if (CPEs[i].CPEMI == nullptr) continue; - if (CPEIsInRange(UserMI, UserOffset, CPEs[i].CPEMI, U.MaxDisp, U.NegOk)) { + if (isCPEntryInRange(UserMI, UserOffset, CPEs[i].CPEMI, U.getMaxDisp(), + U.NegOk)) { DEBUG(dbgs() << "Replacing CPE#" << CPI << " with CPE#" << CPEs[i].CPI << "\n"); // Point the CPUser node to the replacement @@ -1116,7 +1255,7 @@ int ARMConstantIslands::LookForExistingCPEntry(CPUser& U, unsigned UserOffset) CPEs[i].RefCount++; // ...and the original. If we didn't remove the old entry, none of the // addresses changed, so we don't need another pass. - return DecrementOldEntry(CPI, CPEMI) ? 2 : 1; + return decrementCPEReferenceCount(CPI, CPEMI) ? 2 : 1; } } return 0; @@ -1137,7 +1276,7 @@ static inline unsigned getUnconditionalBrDisp(int Opc) { return ((1<<23)-1)*4; } -/// LookForWater - Look for an existing entry in the WaterList in which +/// findAvailableWater - Look for an existing entry in the WaterList in which /// we can place the CPE referenced from U so it's within range of U's MI. /// Returns true if found, false if not. If it returns true, WaterIter /// is set to the WaterList entry. For Thumb, prefer water that will not @@ -1145,13 +1284,13 @@ static inline unsigned getUnconditionalBrDisp(int Opc) { /// terminates, the CPE location for a particular CPUser is only allowed to /// move to a lower address, so search backward from the end of the list and /// prefer the first water that is in range. -bool ARMConstantIslands::LookForWater(CPUser &U, unsigned UserOffset, +bool ARMConstantIslands::findAvailableWater(CPUser &U, unsigned UserOffset, water_iterator &WaterIter) { if (WaterList.empty()) return false; unsigned BestGrowth = ~0u; - for (water_iterator IP = prior(WaterList.end()), B = WaterList.begin();; + for (water_iterator IP = std::prev(WaterList.end()), B = WaterList.begin();; --IP) { MachineBasicBlock* WaterBB = *IP; // Check if water is in range and is either at a lower address than the @@ -1163,9 +1302,10 @@ bool ARMConstantIslands::LookForWater(CPUser &U, unsigned UserOffset, // sure to take advantage of it for all the CPEs near that block, so that // we don't insert more branches than necessary. unsigned Growth; - if (WaterIsInRange(UserOffset, WaterBB, U, Growth) && + if (isWaterInRange(UserOffset, WaterBB, U, Growth) && (WaterBB->getNumber() < U.HighWaterMark->getNumber() || - NewWaterList.count(WaterBB)) && Growth < BestGrowth) { + NewWaterList.count(WaterBB) || WaterBB == U.MI->getParent()) && + Growth < BestGrowth) { // This is the least amount of required padding seen so far. BestGrowth = Growth; WaterIter = IP; @@ -1182,14 +1322,14 @@ bool ARMConstantIslands::LookForWater(CPUser &U, unsigned UserOffset, return BestGrowth != ~0u; } -/// CreateNewWater - No existing WaterList entry will work for +/// createNewWater - No existing WaterList entry will work for /// CPUsers[CPUserIndex], so create a place to put the CPE. The end of the /// block is used if in range, and the conditional branch munged so control /// flow is correct. Otherwise the block is split to create a hole with an /// unconditional branch around it. In either case NewMBB is set to a /// block following which the new island can be inserted (the WaterList /// is not adjusted). -void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex, +void ARMConstantIslands::createNewWater(unsigned CPUserIndex, unsigned UserOffset, MachineBasicBlock *&NewMBB) { CPUser &U = CPUsers[CPUserIndex]; @@ -1202,21 +1342,17 @@ void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex, // If the block does not end in an unconditional branch already, and if the // end of the block is within range, make new water there. (The addition // below is for the unconditional branch we will be adding: 4 bytes on ARM + - // Thumb2, 2 on Thumb1. Possible Thumb1 alignment padding is allowed for - // inside OffsetIsInRange. + // Thumb2, 2 on Thumb1. if (BBHasFallthrough(UserMBB)) { // Size of branch to insert. unsigned Delta = isThumb1 ? 2 : 4; - // End of UserBlock after adding a branch. - unsigned UserBlockEnd = UserBBI.postOffset() + Delta; // Compute the offset where the CPE will begin. - unsigned CPEOffset = WorstCaseAlign(UserBlockEnd, CPELogAlign, - UserBBI.postKnownBits()); + unsigned CPEOffset = UserBBI.postOffset(CPELogAlign) + Delta; - if (OffsetIsInRange(UserOffset, CPEOffset, U)) { + if (isOffsetInRange(UserOffset, CPEOffset, U)) { DEBUG(dbgs() << "Split at end of BB#" << UserMBB->getNumber() << format(", expected CPE offset %#x\n", CPEOffset)); - NewMBB = llvm::next(MachineFunction::iterator(UserMBB)); + NewMBB = &*++UserMBB->getIterator(); // Add an unconditional branch from UserMBB to fallthrough block. Record // it for branch lengthening; this new branch will not get out of range, // but if the preceding conditional branch is out of range, the targets @@ -1231,8 +1367,8 @@ void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex, unsigned MaxDisp = getUnconditionalBrDisp(UncondBr); ImmBranches.push_back(ImmBranch(&UserMBB->back(), MaxDisp, false, UncondBr)); - BBInfo[UserMBB->getNumber()].Size += Delta; - AdjustBBOffsetsAfter(UserMBB); + computeBlockSize(UserMBB); + adjustBBOffsetsAfter(UserMBB); return; } } @@ -1250,23 +1386,19 @@ void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex, // up the insertion point. // Try to split the block so it's fully aligned. Compute the latest split - // point where we can add a 4-byte branch instruction, and then - // WorstCaseAlign to LogAlign. + // point where we can add a 4-byte branch instruction, and then align to + // LogAlign which is the largest possible alignment in the function. unsigned LogAlign = MF->getAlignment(); assert(LogAlign >= CPELogAlign && "Over-aligned constant pool entry"); unsigned KnownBits = UserBBI.internalKnownBits(); unsigned UPad = UnknownPadding(LogAlign, KnownBits); - unsigned BaseInsertOffset = UserOffset + U.MaxDisp; + unsigned BaseInsertOffset = UserOffset + U.getMaxDisp() - UPad; DEBUG(dbgs() << format("Split in middle of big block before %#x", BaseInsertOffset)); - // Account for alignment and unknown padding. - BaseInsertOffset &= ~((1u << LogAlign) - 1); - BaseInsertOffset -= UPad; - // The 4 in the following is for the unconditional branch we'll be inserting // (allows for long branch on Thumb1). Alignment of the island is handled - // inside OffsetIsInRange. + // inside isOffsetInRange. BaseInsertOffset -= 4; DEBUG(dbgs() << format(", adjusted to %#x", BaseInsertOffset) @@ -1278,24 +1410,29 @@ void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex, // pool entries following this block; only the last one is in the water list. // Back past any possible branches (allow for a conditional and a maximally // long unconditional). - if (BaseInsertOffset >= BBInfo[UserMBB->getNumber()+1].Offset) - BaseInsertOffset = BBInfo[UserMBB->getNumber()+1].Offset - - (isThumb1 ? 6 : 8); - unsigned EndInsertOffset = - WorstCaseAlign(BaseInsertOffset + 4, LogAlign, KnownBits) + + if (BaseInsertOffset + 8 >= UserBBI.postOffset()) { + // Ensure BaseInsertOffset is larger than the offset of the instruction + // following UserMI so that the loop which searches for the split point + // iterates at least once. + BaseInsertOffset = + std::max(UserBBI.postOffset() - UPad - 8, + UserOffset + TII->GetInstSizeInBytes(UserMI) + 1); + DEBUG(dbgs() << format("Move inside block: %#x\n", BaseInsertOffset)); + } + unsigned EndInsertOffset = BaseInsertOffset + 4 + UPad + CPEMI->getOperand(2).getImm(); MachineBasicBlock::iterator MI = UserMI; ++MI; unsigned CPUIndex = CPUserIndex+1; unsigned NumCPUsers = CPUsers.size(); - MachineInstr *LastIT = 0; + MachineInstr *LastIT = nullptr; for (unsigned Offset = UserOffset+TII->GetInstSizeInBytes(UserMI); Offset < BaseInsertOffset; - Offset += TII->GetInstSizeInBytes(MI), - MI = llvm::next(MI)) { + Offset += TII->GetInstSizeInBytes(MI), MI = std::next(MI)) { + assert(MI != UserMBB->end() && "Fell off end of block"); if (CPUIndex < NumCPUsers && CPUsers[CPUIndex].MI == MI) { CPUser &U = CPUsers[CPUIndex]; - if (!OffsetIsInRange(Offset, EndInsertOffset, U)) { + if (!isOffsetInRange(Offset, EndInsertOffset, U)) { // Shift intertion point by one unit of alignment so it is within reach. BaseInsertOffset -= 1u << LogAlign; EndInsertOffset -= 1u << LogAlign; @@ -1304,9 +1441,7 @@ void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex, // reused within the block, but it doesn't matter much. Also assume CPEs // are added in order with alignment padding. We may eventually be able // to pack the aligned CPEs better. - EndInsertOffset = RoundUpToAlignment(EndInsertOffset, - 1u << getCPELogAlign(U.CPEMI)) + - U.CPEMI->getOperand(2).getImm(); + EndInsertOffset += U.CPEMI->getOperand(2).getImm(); CPUIndex++; } @@ -1320,30 +1455,34 @@ void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex, // Avoid splitting an IT block. if (LastIT) { unsigned PredReg = 0; - ARMCC::CondCodes CC = llvm::getITInstrPredicate(MI, PredReg); + ARMCC::CondCodes CC = getITInstrPredicate(MI, PredReg); if (CC != ARMCC::AL) MI = LastIT; } - NewMBB = SplitBlockBeforeInstr(MI); + + // We really must not split an IT block. + DEBUG(unsigned PredReg; + assert(!isThumb || getITInstrPredicate(MI, PredReg) == ARMCC::AL)); + + NewMBB = splitBlockBeforeInstr(MI); } -/// HandleConstantPoolUser - Analyze the specified user, checking to see if it +/// handleConstantPoolUser - Analyze the specified user, checking to see if it /// is out-of-range. If so, pick up the constant pool value and move it some /// place in-range. Return true if we changed any addresses (thus must run /// another pass of branch lengthening), false otherwise. -bool ARMConstantIslands::HandleConstantPoolUser(unsigned CPUserIndex) { +bool ARMConstantIslands::handleConstantPoolUser(unsigned CPUserIndex) { CPUser &U = CPUsers[CPUserIndex]; MachineInstr *UserMI = U.MI; MachineInstr *CPEMI = U.CPEMI; - unsigned CPI = CPEMI->getOperand(1).getIndex(); + unsigned CPI = getCombinedIndex(CPEMI); unsigned Size = CPEMI->getOperand(2).getImm(); - // Compute this only once, it's expensive. The 4 or 8 is the value the - // hardware keeps in the PC. - unsigned UserOffset = GetOffsetOf(UserMI) + (isThumb ? 4 : 8); + // Compute this only once, it's expensive. + unsigned UserOffset = getUserOffset(U); // See if the current entry is within range, or there is a clone of it // in range. - int result = LookForExistingCPEntry(U, UserOffset); + int result = findInRangeCPEntry(U, UserOffset); if (result==1) return false; else if (result==2) return true; @@ -1355,31 +1494,29 @@ bool ARMConstantIslands::HandleConstantPoolUser(unsigned CPUserIndex) { MachineBasicBlock *NewIsland = MF->CreateMachineBasicBlock(); MachineBasicBlock *NewMBB; water_iterator IP; - if (LookForWater(U, UserOffset, IP)) { + if (findAvailableWater(U, UserOffset, IP)) { DEBUG(dbgs() << "Found water in range\n"); MachineBasicBlock *WaterBB = *IP; // If the original WaterList entry was "new water" on this iteration, // propagate that to the new island. This is just keeping NewWaterList // updated to match the WaterList, which will be updated below. - if (NewWaterList.count(WaterBB)) { - NewWaterList.erase(WaterBB); + if (NewWaterList.erase(WaterBB)) NewWaterList.insert(NewIsland); - } - // The new CPE goes before the following block (NewMBB). - NewMBB = llvm::next(MachineFunction::iterator(WaterBB)); + // The new CPE goes before the following block (NewMBB). + NewMBB = &*++WaterBB->getIterator(); } else { // No water found. DEBUG(dbgs() << "No water found\n"); - CreateNewWater(CPUserIndex, UserOffset, NewMBB); + createNewWater(CPUserIndex, UserOffset, NewMBB); - // SplitBlockBeforeInstr adds to WaterList, which is important when it is + // splitBlockBeforeInstr adds to WaterList, which is important when it is // called while handling branches so that the water will be seen on the // next iteration for constant pools, but in this context, we don't want // it. Check for this so it will be removed from the WaterList. // Also remove any entry from NewWaterList. - MachineBasicBlock *WaterBB = prior(MachineFunction::iterator(NewMBB)); + MachineBasicBlock *WaterBB = &*--NewMBB->getIterator(); IP = std::find(WaterList.begin(), WaterList.end(), WaterBB); if (IP != WaterList.end()) NewWaterList.erase(WaterBB); @@ -1396,28 +1533,28 @@ bool ARMConstantIslands::HandleConstantPoolUser(unsigned CPUserIndex) { WaterList.erase(IP); // Okay, we know we can put an island before NewMBB now, do it! - MF->insert(NewMBB, NewIsland); + MF->insert(NewMBB->getIterator(), NewIsland); // Update internal data structures to account for the newly inserted MBB. - UpdateForInsertedWaterBlock(NewIsland); - - // Decrement the old entry, and remove it if refcount becomes 0. - DecrementOldEntry(CPI, CPEMI); + updateForInsertedWaterBlock(NewIsland); // Now that we have an island to add the CPE to, clone the original CPE and // add it to the island. U.HighWaterMark = NewIsland; - U.CPEMI = BuildMI(NewIsland, DebugLoc(), TII->get(ARM::CONSTPOOL_ENTRY)) - .addImm(ID).addConstantPoolIndex(CPI).addImm(Size); + U.CPEMI = BuildMI(NewIsland, DebugLoc(), CPEMI->getDesc()) + .addImm(ID).addOperand(CPEMI->getOperand(1)).addImm(Size); CPEntries[CPI].push_back(CPEntry(U.CPEMI, ID, 1)); ++NumCPEs; + // Decrement the old entry, and remove it if refcount becomes 0. + decrementCPEReferenceCount(CPI, CPEMI); + // Mark the basic block as aligned as required by the const-pool entry. NewIsland->setAlignment(getCPELogAlign(U.CPEMI)); // Increase the size of the island block to account for the new entry. BBInfo[NewIsland->getNumber()].Size += Size; - AdjustBBOffsetsAfter(llvm::prior(MachineFunction::iterator(NewIsland))); + adjustBBOffsetsAfter(&*--NewIsland->getIterator()); // Finally, change the CPI in the instruction operand to be ID. for (unsigned i = 0, e = UserMI->getNumOperands(); i != e; ++i) @@ -1432,9 +1569,9 @@ bool ARMConstantIslands::HandleConstantPoolUser(unsigned CPUserIndex) { return true; } -/// RemoveDeadCPEMI - Remove a dead constant pool entry instruction. Update +/// removeDeadCPEMI - Remove a dead constant pool entry instruction. Update /// sizes and offsets of impacted basic blocks. -void ARMConstantIslands::RemoveDeadCPEMI(MachineInstr *CPEMI) { +void ARMConstantIslands::removeDeadCPEMI(MachineInstr *CPEMI) { MachineBasicBlock *CPEBB = CPEMI->getParent(); unsigned Size = CPEMI->getOperand(2).getImm(); CPEMI->eraseFromParent(); @@ -1443,13 +1580,13 @@ void ARMConstantIslands::RemoveDeadCPEMI(MachineInstr *CPEMI) { if (CPEBB->empty()) { BBInfo[CPEBB->getNumber()].Size = 0; - // This block no longer needs to be aligned. . + // This block no longer needs to be aligned. CPEBB->setAlignment(0); } else // Entries are sorted by descending alignment, so realign from the front. CPEBB->setAlignment(getCPELogAlign(CPEBB->begin())); - AdjustBBOffsetsAfter(CPEBB); + adjustBBOffsetsAfter(CPEBB); // An island has only one predecessor BB and one successor BB. Check if // this BB's predecessor jumps directly to this BB's successor. This // shouldn't happen currently. @@ -1457,16 +1594,16 @@ void ARMConstantIslands::RemoveDeadCPEMI(MachineInstr *CPEMI) { // FIXME: remove the empty blocks after all the work is done? } -/// RemoveUnusedCPEntries - Remove constant pool entries whose refcounts +/// removeUnusedCPEntries - Remove constant pool entries whose refcounts /// are zero. -bool ARMConstantIslands::RemoveUnusedCPEntries() { +bool ARMConstantIslands::removeUnusedCPEntries() { unsigned MadeChange = false; for (unsigned i = 0, e = CPEntries.size(); i != e; ++i) { std::vector &CPEs = CPEntries[i]; for (unsigned j = 0, ee = CPEs.size(); j != ee; ++j) { if (CPEs[j].RefCount == 0 && CPEs[j].CPEMI) { - RemoveDeadCPEMI(CPEs[j].CPEMI); - CPEs[j].CPEMI = NULL; + removeDeadCPEMI(CPEs[j].CPEMI); + CPEs[j].CPEMI = nullptr; MadeChange = true; } } @@ -1474,18 +1611,18 @@ bool ARMConstantIslands::RemoveUnusedCPEntries() { return MadeChange; } -/// BBIsInRange - Returns true if the distance between specific MI and +/// isBBInRange - Returns true if the distance between specific MI and /// specific BB can fit in MI's displacement field. -bool ARMConstantIslands::BBIsInRange(MachineInstr *MI,MachineBasicBlock *DestBB, +bool ARMConstantIslands::isBBInRange(MachineInstr *MI,MachineBasicBlock *DestBB, unsigned MaxDisp) { unsigned PCAdj = isThumb ? 4 : 8; - unsigned BrOffset = GetOffsetOf(MI) + PCAdj; + unsigned BrOffset = getOffsetOf(MI) + PCAdj; unsigned DestOffset = BBInfo[DestBB->getNumber()].Offset; DEBUG(dbgs() << "Branch of destination BB#" << DestBB->getNumber() << " from BB#" << MI->getParent()->getNumber() << " max delta=" << MaxDisp - << " from " << GetOffsetOf(MI) << " to " << DestOffset + << " from " << getOffsetOf(MI) << " to " << DestOffset << " offset " << int(DestOffset-BrOffset) << "\t" << *MI); if (BrOffset <= DestOffset) { @@ -1499,37 +1636,37 @@ bool ARMConstantIslands::BBIsInRange(MachineInstr *MI,MachineBasicBlock *DestBB, return false; } -/// FixUpImmediateBr - Fix up an immediate branch whose destination is too far +/// fixupImmediateBr - Fix up an immediate branch whose destination is too far /// away to fit in its displacement field. -bool ARMConstantIslands::FixUpImmediateBr(ImmBranch &Br) { +bool ARMConstantIslands::fixupImmediateBr(ImmBranch &Br) { MachineInstr *MI = Br.MI; MachineBasicBlock *DestBB = MI->getOperand(0).getMBB(); // Check to see if the DestBB is already in-range. - if (BBIsInRange(MI, DestBB, Br.MaxDisp)) + if (isBBInRange(MI, DestBB, Br.MaxDisp)) return false; if (!Br.isCond) - return FixUpUnconditionalBr(Br); - return FixUpConditionalBr(Br); + return fixupUnconditionalBr(Br); + return fixupConditionalBr(Br); } -/// FixUpUnconditionalBr - Fix up an unconditional branch whose destination is +/// fixupUnconditionalBr - Fix up an unconditional branch whose destination is /// too far away to fit in its displacement field. If the LR register has been /// spilled in the epilogue, then we can use BL to implement a far jump. /// Otherwise, add an intermediate branch instruction to a branch. bool -ARMConstantIslands::FixUpUnconditionalBr(ImmBranch &Br) { +ARMConstantIslands::fixupUnconditionalBr(ImmBranch &Br) { MachineInstr *MI = Br.MI; MachineBasicBlock *MBB = MI->getParent(); if (!isThumb1) - llvm_unreachable("FixUpUnconditionalBr is Thumb1 only!"); + llvm_unreachable("fixupUnconditionalBr is Thumb1 only!"); // Use BL to implement far jump. Br.MaxDisp = (1 << 21) * 2; MI->setDesc(TII->get(ARM::tBfar)); BBInfo[MBB->getNumber()].Size += 2; - AdjustBBOffsetsAfter(MBB); + adjustBBOffsetsAfter(MBB); HasFarJump = true; ++NumUBrFixed; @@ -1538,11 +1675,11 @@ ARMConstantIslands::FixUpUnconditionalBr(ImmBranch &Br) { return true; } -/// FixUpConditionalBr - Fix up a conditional branch whose destination is too +/// fixupConditionalBr - Fix up a conditional branch whose destination is too /// far away to fit in its displacement field. It is converted to an inverse /// conditional branch + an unconditional branch to the destination. bool -ARMConstantIslands::FixUpConditionalBr(ImmBranch &Br) { +ARMConstantIslands::fixupConditionalBr(ImmBranch &Br) { MachineInstr *MI = Br.MI; MachineBasicBlock *DestBB = MI->getOperand(0).getMBB(); @@ -1566,7 +1703,7 @@ ARMConstantIslands::FixUpConditionalBr(ImmBranch &Br) { ++NumCBrFixed; if (BMI != MI) { - if (llvm::next(MachineBasicBlock::iterator(MI)) == prior(MBB->end()) && + if (std::next(MachineBasicBlock::iterator(MI)) == std::prev(MBB->end()) && BMI->getOpcode() == Br.UncondBr) { // Last MI in the BB is an unconditional branch. Can we simply invert the // condition and swap destinations: @@ -1576,7 +1713,7 @@ ARMConstantIslands::FixUpConditionalBr(ImmBranch &Br) { // bne L2 // b L1 MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB(); - if (BBIsInRange(MI, NewDest, Br.MaxDisp)) { + if (isBBInRange(MI, NewDest, Br.MaxDisp)) { DEBUG(dbgs() << " Invert Bcc condition and swap its destination with " << *BMI); BMI->getOperand(0).setMBB(DestBB); @@ -1588,7 +1725,7 @@ ARMConstantIslands::FixUpConditionalBr(ImmBranch &Br) { } if (NeedSplit) { - SplitBlockBeforeInstr(MI); + splitBlockBeforeInstr(MI); // No need for the branch to the next block. We're adding an unconditional // branch to the destination. int delta = TII->GetInstSizeInBytes(&MBB->back()); @@ -1596,7 +1733,7 @@ ARMConstantIslands::FixUpConditionalBr(ImmBranch &Br) { MBB->back().eraseFromParent(); // BBInfo[SplitBB].Offset is wrong temporarily, fixed below } - MachineBasicBlock *NextBB = llvm::next(MachineFunction::iterator(MBB)); + MachineBasicBlock *NextBB = &*++MBB->getIterator(); DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber() << " also invert condition and change dest. to BB#" @@ -1620,14 +1757,14 @@ ARMConstantIslands::FixUpConditionalBr(ImmBranch &Br) { // Remove the old conditional branch. It may or may not still be in MBB. BBInfo[MI->getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(MI); MI->eraseFromParent(); - AdjustBBOffsetsAfter(MBB); + adjustBBOffsetsAfter(MBB); return true; } -/// UndoLRSpillRestore - Remove Thumb push / pop instructions that only spills +/// undoLRSpillRestore - Remove Thumb push / pop instructions that only spills /// LR / restores LR to pc. FIXME: This is done here because it's only possible /// to do this if tBfar is not used. -bool ARMConstantIslands::UndoLRSpillRestore() { +bool ARMConstantIslands::undoLRSpillRestore() { bool MadeChange = false; for (unsigned i = 0, e = PushPopMIs.size(); i != e; ++i) { MachineInstr *MI = PushPopMIs[i]; @@ -1646,7 +1783,26 @@ bool ARMConstantIslands::UndoLRSpillRestore() { return MadeChange; } -bool ARMConstantIslands::OptimizeThumb2Instructions() { +// mayOptimizeThumb2Instruction - Returns true if optimizeThumb2Instructions +// below may shrink MI. +bool +ARMConstantIslands::mayOptimizeThumb2Instruction(const MachineInstr *MI) const { + switch(MI->getOpcode()) { + // optimizeThumb2Instructions. + case ARM::t2LEApcrel: + case ARM::t2LDRpci: + // optimizeThumb2Branches. + case ARM::t2B: + case ARM::t2Bcc: + case ARM::tBcc: + // optimizeThumb2JumpTables. + case ARM::t2BR_JT: + return true; + } + return false; +} + +bool ARMConstantIslands::optimizeThumb2Instructions() { bool MadeChange = false; // Shrink ADR and LDR from constantpool. @@ -1677,29 +1833,40 @@ bool ARMConstantIslands::OptimizeThumb2Instructions() { if (!NewOpc) continue; - unsigned UserOffset = GetOffsetOf(U.MI) + 4; + unsigned UserOffset = getUserOffset(U); unsigned MaxOffs = ((1 << Bits) - 1) * Scale; + + // Be conservative with inline asm. + if (!U.KnownAlignment) + MaxOffs -= 2; + // FIXME: Check if offset is multiple of scale if scale is not 4. - if (CPEIsInRange(U.MI, UserOffset, U.CPEMI, MaxOffs, false, true)) { + if (isCPEntryInRange(U.MI, UserOffset, U.CPEMI, MaxOffs, false, true)) { + DEBUG(dbgs() << "Shrink: " << *U.MI); U.MI->setDesc(TII->get(NewOpc)); MachineBasicBlock *MBB = U.MI->getParent(); BBInfo[MBB->getNumber()].Size -= 2; - AdjustBBOffsetsAfter(MBB); + adjustBBOffsetsAfter(MBB); ++NumT2CPShrunk; MadeChange = true; } } - MadeChange |= OptimizeThumb2Branches(); - MadeChange |= OptimizeThumb2JumpTables(); + MadeChange |= optimizeThumb2Branches(); + MadeChange |= optimizeThumb2JumpTables(); return MadeChange; } -bool ARMConstantIslands::OptimizeThumb2Branches() { +bool ARMConstantIslands::optimizeThumb2Branches() { bool MadeChange = false; - for (unsigned i = 0, e = ImmBranches.size(); i != e; ++i) { - ImmBranch &Br = ImmBranches[i]; + // The order in which branches appear in ImmBranches is approximately their + // order within the function body. By visiting later branches first, we reduce + // the distance between earlier forward branches and their targets, making it + // more likely that the cbn?z optimization, which can only apply to forward + // branches, will succeed. + for (unsigned i = ImmBranches.size(); i != 0; --i) { + ImmBranch &Br = ImmBranches[i-1]; unsigned Opcode = Br.MI->getOpcode(); unsigned NewOpc = 0; unsigned Scale = 1; @@ -1721,11 +1888,12 @@ bool ARMConstantIslands::OptimizeThumb2Branches() { if (NewOpc) { unsigned MaxOffs = ((1 << (Bits-1))-1) * Scale; MachineBasicBlock *DestBB = Br.MI->getOperand(0).getMBB(); - if (BBIsInRange(Br.MI, DestBB, MaxOffs)) { + if (isBBInRange(Br.MI, DestBB, MaxOffs)) { + DEBUG(dbgs() << "Shrink branch: " << *Br.MI); Br.MI->setDesc(TII->get(NewOpc)); MachineBasicBlock *MBB = Br.MI->getParent(); BBInfo[MBB->getNumber()].Size -= 2; - AdjustBBOffsetsAfter(MBB); + adjustBBOffsetsAfter(MBB); ++NumT2BrShrunk; MadeChange = true; } @@ -1735,9 +1903,14 @@ bool ARMConstantIslands::OptimizeThumb2Branches() { if (Opcode != ARM::tBcc) continue; + // If the conditional branch doesn't kill CPSR, then CPSR can be liveout + // so this transformation is not safe. + if (!Br.MI->killsRegister(ARM::CPSR)) + continue; + NewOpc = 0; unsigned PredReg = 0; - ARMCC::CondCodes Pred = llvm::getInstrPredicate(Br.MI, PredReg); + ARMCC::CondCodes Pred = getInstrPredicate(Br.MI, PredReg); if (Pred == ARMCC::EQ) NewOpc = ARM::tCBZ; else if (Pred == ARMCC::NE) @@ -1747,7 +1920,7 @@ bool ARMConstantIslands::OptimizeThumb2Branches() { MachineBasicBlock *DestBB = Br.MI->getOperand(0).getMBB(); // Check if the distance is within 126. Subtract starting offset by 2 // because the cmp will be eliminated. - unsigned BrOffset = GetOffsetOf(Br.MI) + 4 - 2; + unsigned BrOffset = getOffsetOf(Br.MI) + 4 - 2; unsigned DestOffset = BBInfo[DestBB->getNumber()].Offset; if (BrOffset < DestOffset && (DestOffset - BrOffset) <= 126) { MachineBasicBlock::iterator CmpMI = Br.MI; @@ -1755,11 +1928,12 @@ bool ARMConstantIslands::OptimizeThumb2Branches() { --CmpMI; if (CmpMI->getOpcode() == ARM::tCMPi8) { unsigned Reg = CmpMI->getOperand(0).getReg(); - Pred = llvm::getInstrPredicate(CmpMI, PredReg); + Pred = getInstrPredicate(CmpMI, PredReg); if (Pred == ARMCC::AL && CmpMI->getOperand(1).getImm() == 0 && isARMLowRegister(Reg)) { MachineBasicBlock *MBB = Br.MI->getParent(); + DEBUG(dbgs() << "Fold: " << *CmpMI << " and: " << *Br.MI); MachineInstr *NewBR = BuildMI(*MBB, CmpMI, Br.MI->getDebugLoc(), TII->get(NewOpc)) .addReg(Reg).addMBB(DestBB,Br.MI->getOperand(0).getTargetFlags()); @@ -1767,7 +1941,7 @@ bool ARMConstantIslands::OptimizeThumb2Branches() { Br.MI->eraseFromParent(); Br.MI = NewBR; BBInfo[MBB->getNumber()].Size -= 2; - AdjustBBOffsetsAfter(MBB); + adjustBBOffsetsAfter(MBB); ++NumCBZ; MadeChange = true; } @@ -1779,29 +1953,145 @@ bool ARMConstantIslands::OptimizeThumb2Branches() { return MadeChange; } -/// OptimizeThumb2JumpTables - Use tbb / tbh instructions to generate smaller +static bool isSimpleIndexCalc(MachineInstr &I, unsigned EntryReg, + unsigned BaseReg) { + if (I.getOpcode() != ARM::t2ADDrs) + return false; + + if (I.getOperand(0).getReg() != EntryReg) + return false; + + if (I.getOperand(1).getReg() != BaseReg) + return false; + + // FIXME: what about CC and IdxReg? + return true; +} + +/// \brief While trying to form a TBB/TBH instruction, we may (if the table +/// doesn't immediately follow the BR_JT) need access to the start of the +/// jump-table. We know one instruction that produces such a register; this +/// function works out whether that definition can be preserved to the BR_JT, +/// possibly by removing an intervening addition (which is usually needed to +/// calculate the actual entry to jump to). +bool ARMConstantIslands::preserveBaseRegister(MachineInstr *JumpMI, + MachineInstr *LEAMI, + unsigned &DeadSize, + bool &CanDeleteLEA, + bool &BaseRegKill) { + if (JumpMI->getParent() != LEAMI->getParent()) + return false; + + // Now we hope that we have at least these instructions in the basic block: + // BaseReg = t2LEA ... + // [...] + // EntryReg = t2ADDrs BaseReg, ... + // [...] + // t2BR_JT EntryReg + // + // We have to be very conservative about what we recognise here though. The + // main perturbing factors to watch out for are: + // + Spills at any point in the chain: not direct problems but we would + // expect a blocking Def of the spilled register so in practice what we + // can do is limited. + // + EntryReg == BaseReg: this is the one situation we should allow a Def + // of BaseReg, but only if the t2ADDrs can be removed. + // + Some instruction other than t2ADDrs computing the entry. Not seen in + // the wild, but we should be careful. + unsigned EntryReg = JumpMI->getOperand(0).getReg(); + unsigned BaseReg = LEAMI->getOperand(0).getReg(); + + CanDeleteLEA = true; + BaseRegKill = false; + MachineInstr *RemovableAdd = nullptr; + MachineBasicBlock::iterator I(LEAMI); + for (++I; &*I != JumpMI; ++I) { + if (isSimpleIndexCalc(*I, EntryReg, BaseReg)) { + RemovableAdd = &*I; + break; + } + + for (unsigned K = 0, E = I->getNumOperands(); K != E; ++K) { + const MachineOperand &MO = I->getOperand(K); + if (!MO.isReg() || !MO.getReg()) + continue; + if (MO.isDef() && MO.getReg() == BaseReg) + return false; + if (MO.isUse() && MO.getReg() == BaseReg) { + BaseRegKill = BaseRegKill || MO.isKill(); + CanDeleteLEA = false; + } + } + } + + if (!RemovableAdd) + return true; + + // Check the add really is removable, and that nothing else in the block + // clobbers BaseReg. + for (++I; &*I != JumpMI; ++I) { + for (unsigned K = 0, E = I->getNumOperands(); K != E; ++K) { + const MachineOperand &MO = I->getOperand(K); + if (!MO.isReg() || !MO.getReg()) + continue; + if (MO.isDef() && MO.getReg() == BaseReg) + return false; + if (MO.isUse() && MO.getReg() == EntryReg) + RemovableAdd = nullptr; + } + } + + if (RemovableAdd) { + RemovableAdd->eraseFromParent(); + DeadSize += 4; + } else if (BaseReg == EntryReg) { + // The add wasn't removable, but clobbered the base for the TBB. So we can't + // preserve it. + return false; + } + + // We reached the end of the block without seeing another definition of + // BaseReg (except, possibly the t2ADDrs, which was removed). BaseReg can be + // used in the TBB/TBH if necessary. + return true; +} + +/// \brief Returns whether CPEMI is the first instruction in the block +/// immediately following JTMI (assumed to be a TBB or TBH terminator). If so, +/// we can switch the first register to PC and usually remove the address +/// calculation that preceded it. +static bool jumpTableFollowsTB(MachineInstr *JTMI, MachineInstr *CPEMI) { + MachineFunction::iterator MBB = JTMI->getParent()->getIterator(); + MachineFunction *MF = MBB->getParent(); + ++MBB; + + return MBB != MF->end() && MBB->begin() != MBB->end() && + &*MBB->begin() == CPEMI; +} + +/// optimizeThumb2JumpTables - Use tbb / tbh instructions to generate smaller /// jumptables when it's possible. -bool ARMConstantIslands::OptimizeThumb2JumpTables() { +bool ARMConstantIslands::optimizeThumb2JumpTables() { bool MadeChange = false; // FIXME: After the tables are shrunk, can we get rid some of the // constantpool tables? MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); - if (MJTI == 0) return false; + if (!MJTI) return false; const std::vector &JT = MJTI->getJumpTables(); for (unsigned i = 0, e = T2JumpTables.size(); i != e; ++i) { MachineInstr *MI = T2JumpTables[i]; const MCInstrDesc &MCID = MI->getDesc(); unsigned NumOps = MCID.getNumOperands(); - unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 3 : 2); + unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 2 : 1); MachineOperand JTOP = MI->getOperand(JTOpIdx); unsigned JTI = JTOP.getIndex(); assert(JTI < JT.size()); bool ByteOk = true; bool HalfWordOk = true; - unsigned JTOffset = GetOffsetOf(MI) + 4; + unsigned JTOffset = getOffsetOf(MI) + 4; const std::vector &JTBBs = JT[JTI].MBBs; for (unsigned j = 0, ee = JTBBs.size(); j != ee; ++j) { MachineBasicBlock *MBB = JTBBs[j]; @@ -1817,107 +2107,98 @@ bool ARMConstantIslands::OptimizeThumb2JumpTables() { break; } - if (ByteOk || HalfWordOk) { - MachineBasicBlock *MBB = MI->getParent(); - unsigned BaseReg = MI->getOperand(0).getReg(); - bool BaseRegKill = MI->getOperand(0).isKill(); - if (!BaseRegKill) - continue; - unsigned IdxReg = MI->getOperand(1).getReg(); - bool IdxRegKill = MI->getOperand(1).isKill(); - - // Scan backwards to find the instruction that defines the base - // register. Due to post-RA scheduling, we can't count on it - // immediately preceding the branch instruction. - MachineBasicBlock::iterator PrevI = MI; - MachineBasicBlock::iterator B = MBB->begin(); - while (PrevI != B && !PrevI->definesRegister(BaseReg)) - --PrevI; - - // If for some reason we didn't find it, we can't do anything, so - // just skip this one. - if (!PrevI->definesRegister(BaseReg)) - continue; + if (!ByteOk && !HalfWordOk) + continue; - MachineInstr *AddrMI = PrevI; - bool OptOk = true; - // Examine the instruction that calculates the jumptable entry address. - // Make sure it only defines the base register and kills any uses - // other than the index register. - for (unsigned k = 0, eee = AddrMI->getNumOperands(); k != eee; ++k) { - const MachineOperand &MO = AddrMI->getOperand(k); - if (!MO.isReg() || !MO.getReg()) - continue; - if (MO.isDef() && MO.getReg() != BaseReg) { - OptOk = false; - break; - } - if (MO.isUse() && !MO.isKill() && MO.getReg() != IdxReg) { - OptOk = false; - break; - } - } - if (!OptOk) - continue; + MachineBasicBlock *MBB = MI->getParent(); + if (!MI->getOperand(0).isKill()) // FIXME: needed now? + continue; + unsigned IdxReg = MI->getOperand(1).getReg(); + bool IdxRegKill = MI->getOperand(1).isKill(); - // Now scan back again to find the tLEApcrel or t2LEApcrelJT instruction - // that gave us the initial base register definition. - for (--PrevI; PrevI != B && !PrevI->definesRegister(BaseReg); --PrevI) - ; + CPUser &User = CPUsers[JumpTableUserIndices[JTI]]; + unsigned DeadSize = 0; + bool CanDeleteLEA = false; + bool BaseRegKill = false; + bool PreservedBaseReg = + preserveBaseRegister(MI, User.MI, DeadSize, CanDeleteLEA, BaseRegKill); - // The instruction should be a tLEApcrel or t2LEApcrelJT; we want - // to delete it as well. - MachineInstr *LeaMI = PrevI; - if ((LeaMI->getOpcode() != ARM::tLEApcrelJT && - LeaMI->getOpcode() != ARM::t2LEApcrelJT) || - LeaMI->getOperand(0).getReg() != BaseReg) - OptOk = false; + if (!jumpTableFollowsTB(MI, User.CPEMI) && !PreservedBaseReg) + continue; - if (!OptOk) - continue; + DEBUG(dbgs() << "Shrink JT: " << *MI); + MachineInstr *CPEMI = User.CPEMI; + unsigned Opc = ByteOk ? ARM::t2TBB_JT : ARM::t2TBH_JT; + MachineBasicBlock::iterator MI_JT = MI; + MachineInstr *NewJTMI = + BuildMI(*MBB, MI_JT, MI->getDebugLoc(), TII->get(Opc)) + .addReg(User.MI->getOperand(0).getReg(), + getKillRegState(BaseRegKill)) + .addReg(IdxReg, getKillRegState(IdxRegKill)) + .addJumpTableIndex(JTI, JTOP.getTargetFlags()) + .addImm(CPEMI->getOperand(0).getImm()); + DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": " << *NewJTMI); + + unsigned JTOpc = ByteOk ? ARM::JUMPTABLE_TBB : ARM::JUMPTABLE_TBH; + CPEMI->setDesc(TII->get(JTOpc)); + + if (jumpTableFollowsTB(MI, User.CPEMI)) { + NewJTMI->getOperand(0).setReg(ARM::PC); + NewJTMI->getOperand(0).setIsKill(false); + + if (CanDeleteLEA) { + User.MI->eraseFromParent(); + DeadSize += 4; + + // The LEA was eliminated, the TBB instruction becomes the only new user + // of the jump table. + User.MI = NewJTMI; + User.MaxDisp = 4; + User.NegOk = false; + User.IsSoImm = false; + User.KnownAlignment = false; + } else { + // The LEA couldn't be eliminated, so we must add another CPUser to + // record the TBB or TBH use. + int CPEntryIdx = JumpTableEntryIndices[JTI]; + auto &CPEs = CPEntries[CPEntryIdx]; + auto Entry = std::find_if(CPEs.begin(), CPEs.end(), [&](CPEntry &E) { + return E.CPEMI == User.CPEMI; + }); + ++Entry->RefCount; + CPUsers.emplace_back(CPUser(NewJTMI, User.CPEMI, 4, false, false)); + } + } - unsigned Opc = ByteOk ? ARM::t2TBB_JT : ARM::t2TBH_JT; - MachineInstr *NewJTMI = BuildMI(MBB, MI->getDebugLoc(), TII->get(Opc)) - .addReg(IdxReg, getKillRegState(IdxRegKill)) - .addJumpTableIndex(JTI, JTOP.getTargetFlags()) - .addImm(MI->getOperand(JTOpIdx+1).getImm()); - // FIXME: Insert an "ALIGN" instruction to ensure the next instruction - // is 2-byte aligned. For now, asm printer will fix it up. - unsigned NewSize = TII->GetInstSizeInBytes(NewJTMI); - unsigned OrigSize = TII->GetInstSizeInBytes(AddrMI); - OrigSize += TII->GetInstSizeInBytes(LeaMI); - OrigSize += TII->GetInstSizeInBytes(MI); - - AddrMI->eraseFromParent(); - LeaMI->eraseFromParent(); - MI->eraseFromParent(); + unsigned NewSize = TII->GetInstSizeInBytes(NewJTMI); + unsigned OrigSize = TII->GetInstSizeInBytes(MI); + MI->eraseFromParent(); - int delta = OrigSize - NewSize; - BBInfo[MBB->getNumber()].Size -= delta; - AdjustBBOffsetsAfter(MBB); + int Delta = OrigSize - NewSize + DeadSize; + BBInfo[MBB->getNumber()].Size -= Delta; + adjustBBOffsetsAfter(MBB); - ++NumTBs; - MadeChange = true; - } + ++NumTBs; + MadeChange = true; } return MadeChange; } -/// ReorderThumb2JumpTables - Adjust the function's block layout to ensure that +/// reorderThumb2JumpTables - Adjust the function's block layout to ensure that /// jump tables always branch forwards, since that's what tbb and tbh need. -bool ARMConstantIslands::ReorderThumb2JumpTables() { +bool ARMConstantIslands::reorderThumb2JumpTables() { bool MadeChange = false; MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); - if (MJTI == 0) return false; + if (!MJTI) return false; const std::vector &JT = MJTI->getJumpTables(); for (unsigned i = 0, e = T2JumpTables.size(); i != e; ++i) { MachineInstr *MI = T2JumpTables[i]; const MCInstrDesc &MCID = MI->getDesc(); unsigned NumOps = MCID.getNumOperands(); - unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 3 : 2); + unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 2 : 1); MachineOperand JTOP = MI->getOperand(JTOpIdx); unsigned JTI = JTOP.getIndex(); assert(JTI < JT.size()); @@ -1935,7 +2216,7 @@ bool ARMConstantIslands::ReorderThumb2JumpTables() { // The destination precedes the switch. Try to move the block forward // so we have a positive offset. MachineBasicBlock *NewBB = - AdjustJTTargetBlockForward(MBB, MI->getParent()); + adjustJTTargetBlockForward(MBB, MI->getParent()); if (NewBB) MJTI->ReplaceMBBInJumpTable(JTI, JTBBs[j], NewBB); MadeChange = true; @@ -1947,17 +2228,16 @@ bool ARMConstantIslands::ReorderThumb2JumpTables() { } MachineBasicBlock *ARMConstantIslands:: -AdjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB) -{ +adjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB) { // If the destination block is terminated by an unconditional branch, // try to move it; otherwise, create a new block following the jump // table that branches back to the actual target. This is a very simple // heuristic. FIXME: We can definitely improve it. - MachineBasicBlock *TBB = 0, *FBB = 0; + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; SmallVector Cond; SmallVector CondPrior; - MachineFunction::iterator BBi = BB; - MachineFunction::iterator OldPrior = prior(BBi); + MachineFunction::iterator BBi = BB->getIterator(); + MachineFunction::iterator OldPrior = std::prev(BBi); // If the block terminator isn't analyzable, don't try to move the block bool B = TII->AnalyzeBranch(*BB, TBB, FBB, Cond); @@ -1973,13 +2253,13 @@ AdjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB) // Update numbering to account for the block being moved. MF->RenumberBlocks(); ++NumJTMoved; - return NULL; + return nullptr; } // Create a new MBB for the code after the jump BB. MachineBasicBlock *NewBB = MF->CreateMachineBasicBlock(JTBB->getBasicBlock()); - MachineFunction::iterator MBBI = JTBB; ++MBBI; + MachineFunction::iterator MBBI = ++JTBB->getIterator(); MF->insert(MBBI, NewBB); // Add an unconditional branch from NewBB to BB. @@ -1994,8 +2274,7 @@ AdjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB) // Update the CFG. NewBB->addSuccessor(BB); - JTBB->removeSuccessor(BB); - JTBB->addSuccessor(NewBB); + JTBB->replaceSuccessor(BB, NewBB); ++NumJTInserted; return NewBB;