X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=blobdiff_plain;f=lib%2FCodeGen%2FBranchFolding.cpp;h=bd89ee366962f30dbd72c1c4050bcc9e3eeaa21f;hp=a2e981680c68191b6b467d7e48eca821e8067a5d;hb=0e59c4e3e8f8e105834d137cccb1e1bb731b5a13;hpb=09fc12a14e15ab617729e47f260879d1dfd2f358 diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp index a2e981680c6..bd89ee36696 100644 --- a/lib/CodeGen/BranchFolding.cpp +++ b/lib/CodeGen/BranchFolding.cpp @@ -12,7 +12,8 @@ // it then removes. // // Note that this pass must be run after register allocation, it cannot handle -// SSA form. +// SSA form. It also must handle virtual registers for targets that emit virtual +// ISA (e.g. NVPTX). // //===----------------------------------------------------------------------===// @@ -20,6 +21,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -95,7 +97,7 @@ bool BranchFolderPass::runOnMachineFunction(MachineFunction &MF) { // TailMerge can create jump into if branches that make CFG irreducible for // HW that requires structurized CFG. bool EnableTailMerge = !MF.getTarget().requiresStructuredCFG() && - PassConfig->getEnableTailMerge(); + PassConfig->getEnableTailMerge(); BranchFolder Folder(EnableTailMerge, /*CommonHoist=*/true, getAnalysis(), getAnalysis()); @@ -132,6 +134,7 @@ void BranchFolder::RemoveDeadBlock(MachineBasicBlock *MBB) { // Remove the block. MF->erase(MBB); + FuncletMembership.erase(MBB); } /// OptimizeImpDefsBlock - If a basic block is just a bunch of implicit_def @@ -150,9 +153,13 @@ bool BranchFolder::OptimizeImpDefsBlock(MachineBasicBlock *MBB) { if (!I->isImplicitDef()) break; unsigned Reg = I->getOperand(0).getReg(); - for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); - SubRegs.isValid(); ++SubRegs) - ImpDefRegs.insert(*SubRegs); + if (TargetRegisterInfo::isPhysicalRegister(Reg)) { + for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); + SubRegs.isValid(); ++SubRegs) + ImpDefRegs.insert(*SubRegs); + } else { + ImpDefRegs.insert(Reg); + } ++I; } if (ImpDefRegs.empty()) @@ -163,8 +170,7 @@ bool BranchFolder::OptimizeImpDefsBlock(MachineBasicBlock *MBB) { if (!TII->isUnpredicatedTerminator(I)) return false; // See if it uses any of the implicitly defined registers. - for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { - MachineOperand &MO = I->getOperand(i); + for (const MachineOperand &MO : I->operands()) { if (!MO.isReg() || !MO.isUse()) continue; unsigned Reg = MO.getReg(); @@ -208,14 +214,17 @@ bool BranchFolder::OptimizeFunction(MachineFunction &MF, // Fix CFG. The later algorithms expect it to be right. bool MadeChange = false; - for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; I++) { - MachineBasicBlock *MBB = I, *TBB = nullptr, *FBB = nullptr; + for (MachineBasicBlock &MBB : MF) { + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; SmallVector Cond; - if (!TII->AnalyzeBranch(*MBB, TBB, FBB, Cond, true)) - MadeChange |= MBB->CorrectExtraCFGEdges(TBB, FBB, !Cond.empty()); - MadeChange |= OptimizeImpDefsBlock(MBB); + if (!TII->AnalyzeBranch(MBB, TBB, FBB, Cond, true)) + MadeChange |= MBB.CorrectExtraCFGEdges(TBB, FBB, !Cond.empty()); + MadeChange |= OptimizeImpDefsBlock(&MBB); } + // Recalculate funclet membership. + FuncletMembership = getFuncletMembership(MF); + bool MadeChangeThisIteration = true; while (MadeChangeThisIteration) { MadeChangeThisIteration = TailMergeBlocks(MF); @@ -235,12 +244,9 @@ bool BranchFolder::OptimizeFunction(MachineFunction &MF, // Walk the function to find jump tables that are live. BitVector JTIsLive(JTI->getJumpTables().size()); - for (MachineFunction::iterator BB = MF.begin(), E = MF.end(); - BB != E; ++BB) { - for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); - I != E; ++I) - for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op) { - MachineOperand &Op = I->getOperand(op); + for (const MachineBasicBlock &BB : MF) { + for (const MachineInstr &I : BB) + for (const MachineOperand &Op : I.operands()) { if (!Op.isJTI()) continue; // Remember that this JT is live. @@ -270,11 +276,17 @@ static unsigned HashMachineInstr(const MachineInstr *MI) { for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { const MachineOperand &Op = MI->getOperand(i); - // Merge in bits from the operand if easy. + // Merge in bits from the operand if easy. We can't use MachineOperand's + // hash_code here because it's not deterministic and we sort by hash value + // later. unsigned OperandHash = 0; switch (Op.getType()) { - case MachineOperand::MO_Register: OperandHash = Op.getReg(); break; - case MachineOperand::MO_Immediate: OperandHash = Op.getImm(); break; + case MachineOperand::MO_Register: + OperandHash = Op.getReg(); + break; + case MachineOperand::MO_Immediate: + OperandHash = Op.getImm(); + break; case MachineOperand::MO_MachineBasicBlock: OperandHash = Op.getMBB()->getNumber(); break; @@ -289,27 +301,20 @@ static unsigned HashMachineInstr(const MachineInstr *MI) { // pull in the offset. OperandHash = Op.getOffset(); break; - default: break; + default: + break; } - Hash += ((OperandHash << 3) | Op.getType()) << (i&31); + Hash += ((OperandHash << 3) | Op.getType()) << (i & 31); } return Hash; } /// HashEndOfMBB - Hash the last instruction in the MBB. static unsigned HashEndOfMBB(const MachineBasicBlock *MBB) { - MachineBasicBlock::const_iterator I = MBB->end(); - if (I == MBB->begin()) - return 0; // Empty MBB. - - --I; - // Skip debug info so it will not affect codegen. - while (I->isDebugValue()) { - if (I==MBB->begin()) - return 0; // MBB empty except for debug info. - --I; - } + MachineBasicBlock::const_iterator I = MBB->getLastNonDebugInstr(); + if (I == MBB->end()) + return 0; return HashMachineInstr(I); } @@ -366,7 +371,7 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1, } // Back past possible debugging pseudos at beginning of block. This matters // when one block differs from the other only by whether debugging pseudos - // are present at the beginning. (This way, the various checks later for + // are present at the beginning. (This way, the various checks later for // I1==MBB1->begin() work as expected.) if (I1 == MBB1->begin() && I2 != MBB2->begin()) { --I2; @@ -427,7 +432,7 @@ MachineBasicBlock *BranchFolder::SplitMBBAt(MachineBasicBlock &CurMBB, MachineFunction &MF = *CurMBB.getParent(); // Create the fall-through block. - MachineFunction::iterator MBBI = &CurMBB; + MachineFunction::iterator MBBI = CurMBB.getIterator(); MachineBasicBlock *NewMBB =MF.CreateMachineBasicBlock(BB); CurMBB.getParent()->insert(++MBBI, NewMBB); @@ -446,6 +451,11 @@ MachineBasicBlock *BranchFolder::SplitMBBAt(MachineBasicBlock &CurMBB, // For targets that use the register scavenger, we must maintain LiveIns. MaintainLiveIns(&CurMBB, NewMBB); + // Add the new block to the funclet. + const auto &FuncletI = FuncletMembership.find(&CurMBB); + if (FuncletI != FuncletMembership.end()) + FuncletMembership[NewMBB] = FuncletI->second; + return NewMBB; } @@ -480,9 +490,18 @@ static void FixTail(MachineBasicBlock *CurMBB, MachineBasicBlock *SuccBB, DebugLoc dl; // FIXME: this is nowhere if (I != MF->end() && !TII->AnalyzeBranch(*CurMBB, TBB, FBB, Cond, true)) { - MachineBasicBlock *NextBB = I; + MachineBasicBlock *NextBB = &*I; if (TBB == NextBB && !Cond.empty() && !FBB) { if (!TII->ReverseBranchCondition(Cond)) { + // XXX-disabled: Don't fold conditional branches that we added + // intentionally. + MachineBasicBlock::iterator I = CurMBB->getLastNonDebugInstr(); + if (I != CurMBB->end()) { + if (I->isConditionalBranch()) { + return; + } + } + TII->RemoveBranch(*CurMBB); TII->InsertBranch(*CurMBB, SuccBB, nullptr, Cond, dl); return; @@ -550,14 +569,23 @@ static unsigned CountTerminators(MachineBasicBlock *MBB, /// and decide if it would be profitable to merge those tails. Return the /// length of the common tail and iterators to the first common instruction /// in each block. -static bool ProfitableToMerge(MachineBasicBlock *MBB1, - MachineBasicBlock *MBB2, - unsigned minCommonTailLength, - unsigned &CommonTailLen, - MachineBasicBlock::iterator &I1, - MachineBasicBlock::iterator &I2, - MachineBasicBlock *SuccBB, - MachineBasicBlock *PredBB) { +static bool +ProfitableToMerge(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2, + unsigned minCommonTailLength, unsigned &CommonTailLen, + MachineBasicBlock::iterator &I1, + MachineBasicBlock::iterator &I2, MachineBasicBlock *SuccBB, + MachineBasicBlock *PredBB, + DenseMap &FuncletMembership) { + // It is never profitable to tail-merge blocks from two different funclets. + if (!FuncletMembership.empty()) { + auto Funclet1 = FuncletMembership.find(MBB1); + assert(Funclet1 != FuncletMembership.end()); + auto Funclet2 = FuncletMembership.find(MBB2); + assert(Funclet2 != FuncletMembership.end()); + if (Funclet1->second != Funclet2->second) + return false; + } + CommonTailLen = ComputeCommonTailLength(MBB1, MBB2, I1, I2); if (CommonTailLen == 0) return false; @@ -601,12 +629,8 @@ static bool ProfitableToMerge(MachineBasicBlock *MBB1, // branch instruction, which is likely to be smaller than the 2 // instructions that would be deleted in the merge. MachineFunction *MF = MBB1->getParent(); - if (EffectiveTailLen >= 2 && - MF->getFunction()->hasFnAttribute(Attribute::OptimizeForSize) && - (I1 == MBB1->begin() || I2 == MBB2->begin())) - return true; - - return false; + return EffectiveTailLen >= 2 && MF->getFunction()->optForSize() && + (I1 == MBB1->begin() || I2 == MBB2->begin()); } /// ComputeSameTails - Look through all the blocks in MergePotentials that have @@ -635,7 +659,8 @@ unsigned BranchFolder::ComputeSameTails(unsigned CurHash, if (ProfitableToMerge(CurMPIter->getBlock(), I->getBlock(), minCommonTailLength, CommonTailLen, TrialBBI1, TrialBBI2, - SuccBB, PredBB)) { + SuccBB, PredBB, + FuncletMembership)) { if (CommonTailLen > maxCommonTailLength) { SameTails.clear(); maxCommonTailLength = CommonTailLen; @@ -728,18 +753,6 @@ bool BranchFolder::CreateCommonTailOnlyBlock(MachineBasicBlock *&PredBB, return true; } -static bool hasIdenticalMMOs(const MachineInstr *MI1, const MachineInstr *MI2) { - auto I1 = MI1->memoperands_begin(), E1 = MI1->memoperands_end(); - auto I2 = MI2->memoperands_begin(), E2 = MI2->memoperands_end(); - if ((E1 - I1) != (E2 - I2)) - return false; - for (; I1 != E1; ++I1, ++I2) { - if (**I1 != **I2) - return false; - } - return true; -} - static void removeMMOsFromMemoryOperations(MachineBasicBlock::iterator MBBIStartPos, MachineBasicBlock &MBBCommon) { @@ -776,8 +789,7 @@ removeMMOsFromMemoryOperations(MachineBasicBlock::iterator MBBIStartPos, assert(MBBICommon->isIdenticalTo(&*MBBI) && "Expected matching MIIs!"); if (MBBICommon->mayLoad() || MBBICommon->mayStore()) - if (!hasIdenticalMMOs(&*MBBI, &*MBBICommon)) - MBBICommon->clearMemRefs(); + MBBICommon->setMemRefs(MBBICommon->mergeMemRefsWith(*MBBI)); ++MBBI; ++MBBICommon; @@ -818,7 +830,7 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB, // Sort by hash value so that blocks with identical end sequences sort // together. - std::stable_sort(MergePotentials.begin(), MergePotentials.end()); + array_pod_sort(MergePotentials.begin(), MergePotentials.end()); // Walk through equivalence sets looking for actual exact matches. while (MergePotentials.size() > 1) { @@ -841,8 +853,8 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB, // block, which we can't jump to), we can treat all blocks with this same // tail at once. Use PredBB if that is one of the possibilities, as that // will not introduce any extra branches. - MachineBasicBlock *EntryBB = MergePotentials.begin()->getBlock()-> - getParent()->begin(); + MachineBasicBlock *EntryBB = + &MergePotentials.front().getBlock()->getParent()->front(); unsigned commonTailIndex = SameTails.size(); // If there are two blocks, check to see if one can be made to fall through // into the other. @@ -918,12 +930,11 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { // First find blocks with no successors. MergePotentials.clear(); - for (MachineFunction::iterator I = MF.begin(), E = MF.end(); - I != E && MergePotentials.size() < TailMergeThreshold; ++I) { - if (TriedMerging.count(I)) - continue; - if (I->succ_empty()) - MergePotentials.push_back(MergePotentialsElt(HashEndOfMBB(I), I)); + for (MachineBasicBlock &MBB : MF) { + if (MergePotentials.size() == TailMergeThreshold) + break; + if (!TriedMerging.count(&MBB) && MBB.succ_empty()) + MergePotentials.push_back(MergePotentialsElt(HashEndOfMBB(&MBB), &MBB)); } // If this is a large problem, avoid visiting the same basic blocks @@ -959,13 +970,13 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { I != E; ++I) { if (I->pred_size() < 2) continue; SmallPtrSet UniquePreds; - MachineBasicBlock *IBB = I; - MachineBasicBlock *PredBB = std::prev(I); + MachineBasicBlock *IBB = &*I; + MachineBasicBlock *PredBB = &*std::prev(I); MergePotentials.clear(); - for (MachineBasicBlock::pred_iterator P = I->pred_begin(), - E2 = I->pred_end(); - P != E2 && MergePotentials.size() < TailMergeThreshold; ++P) { - MachineBasicBlock *PBB = *P; + for (MachineBasicBlock *PBB : I->predecessors()) { + if (MergePotentials.size() == TailMergeThreshold) + break; + if (TriedMerging.count(PBB)) continue; @@ -978,7 +989,7 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { continue; // Skip blocks which may jump to a landing pad. Can't tail merge these. - if (PBB->getLandingPadSuccessor()) + if (PBB->hasEHPadSuccessor()) continue; MachineBasicBlock *TBB = nullptr, *FBB = nullptr; @@ -991,18 +1002,21 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { if (TII->ReverseBranchCondition(NewCond)) continue; // This is the QBB case described above - if (!FBB) - FBB = std::next(MachineFunction::iterator(PBB)); + if (!FBB) { + auto Next = ++PBB->getIterator(); + if (Next != MF.end()) + FBB = &*Next; + } } // Failing case: the only way IBB can be reached from PBB is via // exception handling. Happens for landing pads. Would be nice to have // a bit in the edge so we didn't have to do all this. - if (IBB->isLandingPad()) { - MachineFunction::iterator IP = PBB; IP++; + if (IBB->isEHPad()) { + MachineFunction::iterator IP = ++PBB->getIterator(); MachineBasicBlock *PredNextBB = nullptr; if (IP != MF.end()) - PredNextBB = IP; + PredNextBB = &*IP; if (!TBB) { if (IBB != PredNextBB) // fallthrough continue; @@ -1028,7 +1042,7 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { NewCond, dl); } - MergePotentials.push_back(MergePotentialsElt(HashEndOfMBB(PBB), *P)); + MergePotentials.push_back(MergePotentialsElt(HashEndOfMBB(PBB), PBB)); } } @@ -1043,7 +1057,7 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { // Reinsert an unconditional branch if needed. The 1 below can occur as a // result of removing blocks in TryTailMergeBlocks. - PredBB = std::prev(I); // this may have been changed in TryTailMergeBlocks + PredBB = &*std::prev(I); // this may have been changed in TryTailMergeBlocks if (MergePotentials.size() == 1 && MergePotentials.begin()->getBlock() != PredBB) FixTail(MergePotentials.begin()->getBlock(), IBB, TII); @@ -1081,13 +1095,19 @@ void BranchFolder::setCommonTailEdgeWeights(MachineBasicBlock &TailMBB) { if (TailMBB.succ_size() <= 1) return; - auto MaxEdgeFreq = *std::max_element(EdgeFreqLs.begin(), EdgeFreqLs.end()); - uint64_t Scale = MaxEdgeFreq.getFrequency() / UINT32_MAX + 1; + auto SumEdgeFreq = + std::accumulate(EdgeFreqLs.begin(), EdgeFreqLs.end(), BlockFrequency(0)) + .getFrequency(); auto EdgeFreq = EdgeFreqLs.begin(); - for (auto SuccI = TailMBB.succ_begin(), SuccE = TailMBB.succ_end(); - SuccI != SuccE; ++SuccI, ++EdgeFreq) - TailMBB.setSuccWeight(SuccI, EdgeFreq->getFrequency() / Scale); + if (SumEdgeFreq > 0) { + for (auto SuccI = TailMBB.succ_begin(), SuccE = TailMBB.succ_end(); + SuccI != SuccE; ++SuccI, ++EdgeFreq) { + auto Prob = BranchProbability::getBranchProbability( + EdgeFreq->getFrequency(), SumEdgeFreq); + TailMBB.setSuccProbability(SuccI, Prob); + } + } } //===----------------------------------------------------------------------===// @@ -1099,10 +1119,12 @@ bool BranchFolder::OptimizeBranches(MachineFunction &MF) { // Make sure blocks are numbered in order MF.RenumberBlocks(); + // Renumbering blocks alters funclet membership, recalculate it. + FuncletMembership = getFuncletMembership(MF); for (MachineFunction::iterator I = std::next(MF.begin()), E = MF.end(); I != E; ) { - MachineBasicBlock *MBB = I++; + MachineBasicBlock *MBB = &*I++; MadeChange |= OptimizeBlock(MBB); // If it is dead, remove it. @@ -1112,31 +1134,22 @@ bool BranchFolder::OptimizeBranches(MachineFunction &MF) { ++NumDeadBlocks; } } + return MadeChange; } // Blocks should be considered empty if they contain only debug info; // else the debug info would affect codegen. static bool IsEmptyBlock(MachineBasicBlock *MBB) { - if (MBB->empty()) - return true; - for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end(); - MBBI!=MBBE; ++MBBI) { - if (!MBBI->isDebugValue()) - return false; - } - return true; + return MBB->getFirstNonDebugInstr() == MBB->end(); } // Blocks with only debug info and branches should be considered the same // as blocks with only branches. static bool IsBranchOnlyBlock(MachineBasicBlock *MBB) { - MachineBasicBlock::iterator MBBI, MBBE; - for (MBBI = MBB->begin(), MBBE = MBB->end(); MBBI!=MBBE; ++MBBI) { - if (!MBBI->isDebugValue()) - break; - } - return (MBBI->isBranch()); + MachineBasicBlock::iterator I = MBB->getFirstNonDebugInstr(); + assert(I != MBB->end() && "empty block!"); + return I->isBranch(); } /// IsBetterFallthrough - Return true if it would be clearly better to @@ -1149,36 +1162,24 @@ static bool IsBetterFallthrough(MachineBasicBlock *MBB1, // MBB1 doesn't, we prefer to fall through into MBB1. This allows us to // optimize branches that branch to either a return block or an assert block // into a fallthrough to the return. - if (IsEmptyBlock(MBB1) || IsEmptyBlock(MBB2)) return false; + MachineBasicBlock::iterator MBB1I = MBB1->getLastNonDebugInstr(); + MachineBasicBlock::iterator MBB2I = MBB2->getLastNonDebugInstr(); + if (MBB1I == MBB1->end() || MBB2I == MBB2->end()) + return false; // If there is a clear successor ordering we make sure that one block // will fall through to the next if (MBB1->isSuccessor(MBB2)) return true; if (MBB2->isSuccessor(MBB1)) return false; - // Neither block consists entirely of debug info (per IsEmptyBlock check), - // so we needn't test for falling off the beginning here. - MachineBasicBlock::iterator MBB1I = --MBB1->end(); - while (MBB1I->isDebugValue()) - --MBB1I; - MachineBasicBlock::iterator MBB2I = --MBB2->end(); - while (MBB2I->isDebugValue()) - --MBB2I; return MBB2I->isCall() && !MBB1I->isCall(); } /// getBranchDebugLoc - Find and return, if any, the DebugLoc of the branch -/// instructions on the block. Always use the DebugLoc of the first -/// branching instruction found unless its absent, in which case use the -/// DebugLoc of the second if present. +/// instructions on the block. static DebugLoc getBranchDebugLoc(MachineBasicBlock &MBB) { - MachineBasicBlock::iterator I = MBB.end(); - if (I == MBB.begin()) - return DebugLoc(); - --I; - while (I->isDebugValue() && I != MBB.begin()) - --I; - if (I->isBranch()) + MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); + if (I != MBB.end() && I->isBranch()) return I->getDebugLoc(); return DebugLoc(); } @@ -1190,30 +1191,46 @@ bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) { MachineFunction &MF = *MBB->getParent(); ReoptimizeBlock: - MachineFunction::iterator FallThrough = MBB; + MachineFunction::iterator FallThrough = MBB->getIterator(); ++FallThrough; + // Make sure MBB and FallThrough belong to the same funclet. + bool SameFunclet = true; + if (!FuncletMembership.empty() && FallThrough != MF.end()) { + auto MBBFunclet = FuncletMembership.find(MBB); + assert(MBBFunclet != FuncletMembership.end()); + auto FallThroughFunclet = FuncletMembership.find(&*FallThrough); + assert(FallThroughFunclet != FuncletMembership.end()); + SameFunclet = MBBFunclet->second == FallThroughFunclet->second; + } + // If this block is empty, make everyone use its fall-through, not the block // explicitly. Landing pads should not do this since the landing-pad table // points to this block. Blocks with their addresses taken shouldn't be // optimized away. - if (IsEmptyBlock(MBB) && !MBB->isLandingPad() && !MBB->hasAddressTaken()) { + if (IsEmptyBlock(MBB) && !MBB->isEHPad() && !MBB->hasAddressTaken() && + SameFunclet) { // Dead block? Leave for cleanup later. if (MBB->pred_empty()) return MadeChange; if (FallThrough == MF.end()) { // TODO: Simplify preds to not branch here if possible! + } else if (FallThrough->isEHPad()) { + // Don't rewrite to a landing pad fallthough. That could lead to the case + // where a BB jumps to more than one landing pad. + // TODO: Is it ever worth rewriting predecessors which don't already + // jump to a landing pad, and so can safely jump to the fallthrough? } else { // Rewrite all predecessors of the old block to go to the fallthrough // instead. while (!MBB->pred_empty()) { MachineBasicBlock *Pred = *(MBB->pred_end()-1); - Pred->ReplaceUsesOfBlockWith(MBB, FallThrough); + Pred->ReplaceUsesOfBlockWith(MBB, &*FallThrough); } // If MBB was the target of a jump table, update jump tables to go to the // fallthrough instead. if (MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) - MJTI->ReplaceMBBInJumpTables(MBB, FallThrough); + MJTI->ReplaceMBBInJumpTables(MBB, &*FallThrough); MadeChange = true; } return MadeChange; @@ -1255,7 +1272,7 @@ ReoptimizeBlock: // AnalyzeBranch. if (PriorCond.empty() && !PriorTBB && MBB->pred_size() == 1 && PrevBB.succ_size() == 1 && - !MBB->hasAddressTaken() && !MBB->isLandingPad()) { + !MBB->hasAddressTaken() && !MBB->isEHPad()) { DEBUG(dbgs() << "\nMerging into block: " << PrevBB << "From MBB: " << *MBB); // Remove redundant DBG_VALUEs first. @@ -1285,6 +1302,15 @@ ReoptimizeBlock: // If the previous branch *only* branches to *this* block (conditional or // not) remove the branch. if (PriorTBB == MBB && !PriorFBB) { + // XXX-disabled: Don't fold conditional branches that we added + // intentionally. + MachineBasicBlock::iterator I = PrevBB.getLastNonDebugInstr(); + if (I != PrevBB.end()) { + if (I->isConditionalBranch()) { + return MadeChange ; + } + } + TII->RemoveBranch(PrevBB); MadeChange = true; ++NumBranchOpts; @@ -1294,6 +1320,15 @@ ReoptimizeBlock: // If the prior block branches somewhere else on the condition and here if // the condition is false, remove the uncond second branch. if (PriorFBB == MBB) { + // XXX-disabled: Don't fold conditional branches that we added + // intentionally. + MachineBasicBlock::iterator I = PrevBB.getLastNonDebugInstr(); + if (I != PrevBB.end()) { + if (I->isConditionalBranch()) { + return MadeChange ; + } + } + DebugLoc dl = getBranchDebugLoc(PrevBB); TII->RemoveBranch(PrevBB); TII->InsertBranch(PrevBB, PriorTBB, nullptr, PriorCond, dl); @@ -1306,6 +1341,15 @@ ReoptimizeBlock: // if the branch condition is reversible, reverse the branch to create a // fall-through. if (PriorTBB == MBB) { + // XXX-disabled: Don't fold conditional branches that we added + // intentionally. + MachineBasicBlock::iterator I = PrevBB.getLastNonDebugInstr(); + if (I != PrevBB.end()) { + if (I->isConditionalBranch()) { + return MadeChange ; + } + } + SmallVector NewPriorCond(PriorCond); if (!TII->ReverseBranchCondition(NewPriorCond)) { DebugLoc dl = getBranchDebugLoc(PrevBB); @@ -1351,7 +1395,7 @@ ReoptimizeBlock: TII->InsertBranch(PrevBB, MBB, nullptr, NewPriorCond, dl); // Move this block to the end of the function. - MBB->moveAfter(--MF.end()); + MBB->moveAfter(&MF.back()); MadeChange = true; ++NumBranchOpts; return MadeChange; @@ -1389,7 +1433,7 @@ ReoptimizeBlock: // other blocks across it. if (CurTBB && CurCond.empty() && !CurFBB && IsBranchOnlyBlock(MBB) && CurTBB != MBB && - !MBB->hasAddressTaken()) { + !MBB->hasAddressTaken() && !MBB->isEHPad()) { DebugLoc dl = getBranchDebugLoc(*MBB); // This block may contain just an unconditional branch. Because there can // be 'non-branch terminators' in the block, try removing the branch and @@ -1398,19 +1442,10 @@ ReoptimizeBlock: // If the only things remaining in the block are debug info, remove these // as well, so this will behave the same as an empty block in non-debug // mode. - if (!MBB->empty()) { - bool NonDebugInfoFound = false; - for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); - I != E; ++I) { - if (!I->isDebugValue()) { - NonDebugInfoFound = true; - break; - } - } - if (!NonDebugInfoFound) - // Make the block empty, losing the debug info (we could probably - // improve this in some cases.) - MBB->erase(MBB->begin(), MBB->end()); + if (IsEmptyBlock(MBB)) { + // Make the block empty, losing the debug info (we could probably + // improve this in some cases.) + MBB->erase(MBB->begin(), MBB->end()); } // If this block is just an unconditional branch to CurTBB, we can // usually completely eliminate the block. The only case we cannot @@ -1495,14 +1530,11 @@ ReoptimizeBlock: // see if it has a fall-through into its successor. bool CurFallsThru = MBB->canFallThrough(); - if (!MBB->isLandingPad()) { + if (!MBB->isEHPad()) { // Check all the predecessors of this block. If one of them has no fall // throughs, move this block right after it. - for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), - E = MBB->pred_end(); PI != E; ++PI) { + for (MachineBasicBlock *PredBB : MBB->predecessors()) { // Analyze the branch at the end of the pred. - MachineBasicBlock *PredBB = *PI; - MachineFunction::iterator PredFallthrough = PredBB; ++PredFallthrough; MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr; SmallVector PredCond; if (PredBB != MBB && !PredBB->canFallThrough() && @@ -1520,8 +1552,7 @@ ReoptimizeBlock: // B elsewhere // next: if (CurFallsThru) { - MachineBasicBlock *NextBB = - std::next(MachineFunction::iterator(MBB)); + MachineBasicBlock *NextBB = &*std::next(MBB->getIterator()); CurCond.clear(); TII->InsertBranch(*MBB, NextBB, nullptr, CurCond, DebugLoc()); } @@ -1534,11 +1565,9 @@ ReoptimizeBlock: if (!CurFallsThru) { // Check all successors to see if we can move this block before it. - for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), - E = MBB->succ_end(); SI != E; ++SI) { + for (MachineBasicBlock *SuccBB : MBB->successors()) { // Analyze the branch at the end of the block before the succ. - MachineBasicBlock *SuccBB = *SI; - MachineFunction::iterator SuccPrev = SuccBB; --SuccPrev; + MachineFunction::iterator SuccPrev = --SuccBB->getIterator(); // If this block doesn't already fall-through to that successor, and if // the succ doesn't already have a block that can fall through into it, @@ -1546,7 +1575,7 @@ ReoptimizeBlock: // fallthrough to happen. if (SuccBB != MBB && &*SuccPrev != MBB && !SuccPrev->canFallThrough() && !CurUnAnalyzable && - !SuccBB->isLandingPad()) { + !SuccBB->isEHPad()) { MBB->moveBefore(SuccBB); MadeChange = true; goto ReoptimizeBlock; @@ -1558,10 +1587,18 @@ ReoptimizeBlock: // removed, move this block to the end of the function. MachineBasicBlock *PrevTBB = nullptr, *PrevFBB = nullptr; SmallVector PrevCond; + // We're looking for cases where PrevBB could possibly fall through to + // FallThrough, but if FallThrough is an EH pad that wouldn't be useful + // so here we skip over any EH pads so we might have a chance to find + // a branch target from PrevBB. + while (FallThrough != MF.end() && FallThrough->isEHPad()) + ++FallThrough; + // Now check to see if the current block is sitting between PrevBB and + // a block to which it could fall through. if (FallThrough != MF.end() && !TII->AnalyzeBranch(PrevBB, PrevTBB, PrevFBB, PrevCond, true) && - PrevBB.isSuccessor(FallThrough)) { - MBB->moveAfter(--MF.end()); + PrevBB.isSuccessor(&*FallThrough)) { + MBB->moveAfter(&MF.back()); MadeChange = true; return MadeChange; } @@ -1580,7 +1617,7 @@ ReoptimizeBlock: bool BranchFolder::HoistCommonCode(MachineFunction &MF) { bool MadeChange = false; for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ) { - MachineBasicBlock *MBB = I++; + MachineBasicBlock *MBB = &*I++; MadeChange |= HoistCommonCodeInSuccs(MBB); } @@ -1591,15 +1628,23 @@ bool BranchFolder::HoistCommonCode(MachineFunction &MF) { /// its 'true' successor. static MachineBasicBlock *findFalseBlock(MachineBasicBlock *BB, MachineBasicBlock *TrueBB) { - for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), - E = BB->succ_end(); SI != E; ++SI) { - MachineBasicBlock *SuccBB = *SI; + for (MachineBasicBlock *SuccBB : BB->successors()) if (SuccBB != TrueBB) return SuccBB; - } return nullptr; } +template +static void addRegAndItsAliases(unsigned Reg, const TargetRegisterInfo *TRI, + Container &Set) { + if (TargetRegisterInfo::isPhysicalRegister(Reg)) { + for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) + Set.insert(*AI); + } else { + Set.insert(Reg); + } +} + /// findHoistingInsertPosAndDeps - Find the location to move common instructions /// in successors to. The location is usually just before the terminator, /// however if the terminator is a conditional branch and its previous @@ -1617,16 +1662,14 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB, if (!TII->isUnpredicatedTerminator(Loc)) return MBB->end(); - for (unsigned i = 0, e = Loc->getNumOperands(); i != e; ++i) { - const MachineOperand &MO = Loc->getOperand(i); + for (const MachineOperand &MO : Loc->operands()) { if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!Reg) continue; if (MO.isUse()) { - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - Uses.insert(*AI); + addRegAndItsAliases(Reg, TRI, Uses); } else { if (!MO.isDead()) // Don't try to hoist code in the rare case the terminator defines a @@ -1635,8 +1678,7 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB, // If the terminator defines a register, make sure we don't hoist // the instruction whose def might be clobbered by the terminator. - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - Defs.insert(*AI); + addRegAndItsAliases(Reg, TRI, Defs); } } @@ -1653,8 +1695,7 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB, --PI; bool IsDef = false; - for (unsigned i = 0, e = PI->getNumOperands(); !IsDef && i != e; ++i) { - const MachineOperand &MO = PI->getOperand(i); + for (const MachineOperand &MO : PI->operands()) { // If PI has a regmask operand, it is probably a call. Separate away. if (MO.isRegMask()) return Loc; @@ -1663,8 +1704,10 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB, unsigned Reg = MO.getReg(); if (!Reg) continue; - if (Uses.count(Reg)) + if (Uses.count(Reg)) { IsDef = true; + break; + } } if (!IsDef) // The condition setting instruction is not just before the conditional @@ -1678,30 +1721,28 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB, // Also avoid moving code above predicated instruction since it's hard to // reason about register liveness with predicated instruction. bool DontMoveAcrossStore = true; - if (!PI->isSafeToMove(TII, nullptr, DontMoveAcrossStore) || - TII->isPredicated(PI)) + if (!PI->isSafeToMove(nullptr, DontMoveAcrossStore) || TII->isPredicated(PI)) return MBB->end(); // Find out what registers are live. Note this routine is ignoring other live // registers which are only used by instructions in successor blocks. - for (unsigned i = 0, e = PI->getNumOperands(); i != e; ++i) { - const MachineOperand &MO = PI->getOperand(i); + for (const MachineOperand &MO : PI->operands()) { if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!Reg) continue; if (MO.isUse()) { - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - Uses.insert(*AI); + addRegAndItsAliases(Reg, TRI, Uses); } else { if (Uses.erase(Reg)) { - for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) - Uses.erase(*SubRegs); // Use sub-registers to be conservative + if (TargetRegisterInfo::isPhysicalRegister(Reg)) { + for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) + Uses.erase(*SubRegs); // Use sub-registers to be conservative + } } - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - Defs.insert(*AI); + addRegAndItsAliases(Reg, TRI, Defs); } } @@ -1765,8 +1806,7 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) { break; bool IsSafe = true; - for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) { - MachineOperand &MO = TIB->getOperand(i); + for (MachineOperand &MO : TIB->operands()) { // Don't attempt to hoist instructions with register masks. if (MO.isRegMask()) { IsSafe = false; @@ -1817,32 +1857,33 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) { break; bool DontMoveAcrossStore = true; - if (!TIB->isSafeToMove(TII, nullptr, DontMoveAcrossStore)) + if (!TIB->isSafeToMove(nullptr, DontMoveAcrossStore)) break; // Remove kills from LocalDefsSet, these registers had short live ranges. - for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) { - MachineOperand &MO = TIB->getOperand(i); + for (const MachineOperand &MO : TIB->operands()) { if (!MO.isReg() || !MO.isUse() || !MO.isKill()) continue; unsigned Reg = MO.getReg(); if (!Reg || !LocalDefsSet.count(Reg)) continue; - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - LocalDefsSet.erase(*AI); + if (TargetRegisterInfo::isPhysicalRegister(Reg)) { + for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) + LocalDefsSet.erase(*AI); + } else { + LocalDefsSet.erase(Reg); + } } // Track local defs so we can update liveins. - for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) { - MachineOperand &MO = TIB->getOperand(i); + for (const MachineOperand &MO : TIB->operands()) { if (!MO.isReg() || !MO.isDef() || MO.isDead()) continue; unsigned Reg = MO.getReg(); if (!Reg) continue; LocalDefs.push_back(Reg); - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - LocalDefsSet.insert(*AI); + addRegAndItsAliases(Reg, TRI, LocalDefsSet); } HasDups = true;