// - Optimize Loads:
//
// Loads that can be folded into a later instruction. A load is foldable
-// if it loads to virtual registers and the virtual register defined has
+// if it loads to virtual registers and the virtual register defined has
// a single use.
//
// - Optimize Copies and Bitcast (more generally, target specific copies):
DisableAdvCopyOpt("disable-adv-copy-opt", cl::Hidden, cl::init(false),
cl::desc("Disable advanced copy optimization"));
+static cl::opt<bool> DisableNAPhysCopyOpt(
+ "disable-non-allocatable-phys-copy-opt", cl::Hidden, cl::init(false),
+ cl::desc("Disable non-allocatable physical register copy optimization"));
+
+// Limit the number of PHI instructions to process
+// in PeepholeOptimizer::getNextSource.
+static cl::opt<unsigned> RewritePHILimit(
+ "rewrite-phi-limit", cl::Hidden, cl::init(10),
+ cl::desc("Limit the length of PHI chains to lookup"));
+
STATISTIC(NumReuse, "Number of extension results reused");
STATISTIC(NumCmps, "Number of compares eliminated");
STATISTIC(NumImmFold, "Number of move immediate folded");
STATISTIC(NumSelects, "Number of selects optimized");
STATISTIC(NumUncoalescableCopies, "Number of uncoalescable copies optimized");
STATISTIC(NumRewrittenCopies, "Number of copies rewritten");
+STATISTIC(NumNAPhysCopies, "Number of non-allocatable physical copies removed");
namespace {
class ValueTrackerResult;
}
}
+ /// \brief Track Def -> Use info used for rewriting copies.
+ typedef SmallDenseMap<TargetInstrInfo::RegSubRegPair, ValueTrackerResult>
+ RewriteMapTy;
+
private:
bool optimizeCmpInstr(MachineInstr *MI, MachineBasicBlock *MBB);
bool optimizeExtInstr(MachineInstr *MI, MachineBasicBlock *MBB,
bool optimizeSelect(MachineInstr *MI,
SmallPtrSetImpl<MachineInstr *> &LocalMIs);
bool optimizeCondBranch(MachineInstr *MI);
- bool optimizeCopyOrBitcast(MachineInstr *MI);
bool optimizeCoalescableCopy(MachineInstr *MI);
bool optimizeUncoalescableCopy(MachineInstr *MI,
SmallPtrSetImpl<MachineInstr *> &LocalMIs);
- bool findNextSource(unsigned &Reg, unsigned &SubReg);
+ bool findNextSource(unsigned Reg, unsigned SubReg,
+ RewriteMapTy &RewriteMap);
bool isMoveImmediate(MachineInstr *MI,
SmallSet<unsigned, 4> &ImmDefRegs,
DenseMap<unsigned, MachineInstr*> &ImmDefMIs);
bool foldImmediate(MachineInstr *MI, MachineBasicBlock *MBB,
SmallSet<unsigned, 4> &ImmDefRegs,
DenseMap<unsigned, MachineInstr*> &ImmDefMIs);
+
+ /// \brief If copy instruction \p MI is a virtual register copy, track it in
+ /// the set \p CopySrcRegs and \p CopyMIs. If this virtual register was
+ /// previously seen as a copy, replace the uses of this copy with the
+ /// previously seen copy's destination register.
+ bool foldRedundantCopy(MachineInstr *MI,
+ SmallSet<unsigned, 4> &CopySrcRegs,
+ DenseMap<unsigned, MachineInstr *> &CopyMIs);
+
+ /// \brief Is the register \p Reg a non-allocatable physical register?
+ bool isNAPhysCopy(unsigned Reg);
+
+ /// \brief If copy instruction \p MI is a non-allocatable virtual<->physical
+ /// register copy, track it in the \p NAPhysToVirtMIs map. If this
+ /// non-allocatable physical register was previously copied to a virtual
+ /// registered and hasn't been clobbered, the virt->phys copy can be
+ /// deleted.
+ bool foldRedundantNAPhysCopy(
+ MachineInstr *MI,
+ DenseMap<unsigned, MachineInstr *> &NAPhysToVirtMIs);
+
bool isLoadFoldable(MachineInstr *MI,
SmallSet<unsigned, 16> &FoldAsLoadDefCandidates);
assert(Idx < getNumSources() && "SubReg source out of index");
return RegSrcs[Idx].SubReg;
}
+
+ bool operator==(const ValueTrackerResult &Other) {
+ if (Other.getInst() != getInst())
+ return false;
+
+ if (Other.getNumSources() != getNumSources())
+ return false;
+
+ for (int i = 0, e = Other.getNumSources(); i != e; ++i)
+ if (Other.getSrcReg(i) != getSrcReg(i) ||
+ Other.getSrcSubReg(i) != getSrcSubReg(i))
+ return false;
+ return true;
+ }
};
/// \brief Helper class to track the possible sources of a value defined by
/// \brief Specialized version of getNextSource for SubregToReg
/// instructions.
ValueTrackerResult getNextSourceFromSubregToReg();
+ /// \brief Specialized version of getNextSource for PHI instructions.
+ ValueTrackerResult getNextSourceFromPHI();
public:
/// \brief Create a ValueTracker instance for the value defined by \p Reg.
INITIALIZE_PASS_END(PeepholeOptimizer, "peephole-opts",
"Peephole Optimizations", false, false)
-/// optimizeExtInstr - If instruction is a copy-like instruction, i.e. it reads
-/// a single register and writes a single register and it does not modify the
-/// source, and if the source value is preserved as a sub-register of the
-/// result, then replace all reachable uses of the source with the subreg of the
-/// result.
+/// If instruction is a copy-like instruction, i.e. it reads a single register
+/// and writes a single register and it does not modify the source, and if the
+/// source value is preserved as a sub-register of the result, then replace all
+/// reachable uses of the source with the subreg of the result.
///
/// Do not generate an EXTRACT that is used only in a debug use, as this changes
/// the code. Since this code does not currently share EXTRACTs, just ignore all
return Changed;
}
-/// optimizeCmpInstr - If the instruction is a compare and the previous
-/// instruction it's comparing against all ready sets (or could be modified to
-/// set) the same flag as the compare, then we can remove the comparison and use
-/// the flag from the previous instruction.
+/// If the instruction is a compare and the previous instruction it's comparing
+/// against already sets (or could be modified to set) the same flag as the
+/// compare, then we can remove the comparison and use the flag from the
+/// previous instruction.
bool PeepholeOptimizer::optimizeCmpInstr(MachineInstr *MI,
MachineBasicBlock *MBB) {
// If this instruction is a comparison against zero and isn't comparing a
return TII->optimizeCondBranch(MI);
}
-/// \brief Check if the registers defined by the pair (RegisterClass, SubReg)
-/// share the same register file.
-static bool shareSameRegisterFile(const TargetRegisterInfo &TRI,
- const TargetRegisterClass *DefRC,
- unsigned DefSubReg,
- const TargetRegisterClass *SrcRC,
- unsigned SrcSubReg) {
- // Same register class.
- if (DefRC == SrcRC)
- return true;
-
- // Both operands are sub registers. Check if they share a register class.
- unsigned SrcIdx, DefIdx;
- if (SrcSubReg && DefSubReg)
- return TRI.getCommonSuperRegClass(SrcRC, SrcSubReg, DefRC, DefSubReg,
- SrcIdx, DefIdx) != nullptr;
- // At most one of the register is a sub register, make it Src to avoid
- // duplicating the test.
- if (!SrcSubReg) {
- std::swap(DefSubReg, SrcSubReg);
- std::swap(DefRC, SrcRC);
- }
-
- // One of the register is a sub register, check if we can get a superclass.
- if (SrcSubReg)
- return TRI.getMatchingSuperRegClass(SrcRC, DefRC, SrcSubReg) != nullptr;
- // Plain copy.
- return TRI.getCommonSubClass(DefRC, SrcRC) != nullptr;
-}
-
/// \brief Try to find the next source that share the same register file
/// for the value defined by \p Reg and \p SubReg.
-/// When true is returned, \p Reg and \p SubReg are updated with the
-/// register number and sub-register index of the new source.
+/// When true is returned, the \p RewriteMap can be used by the client to
+/// retrieve all Def -> Use along the way up to the next source. Any found
+/// Use that is not itself a key for another entry, is the next source to
+/// use. During the search for the next source, multiple sources can be found
+/// given multiple incoming sources of a PHI instruction. In this case, we
+/// look in each PHI source for the next source; all found next sources must
+/// share the same register file as \p Reg and \p SubReg. The client should
+/// then be capable to rewrite all intermediate PHIs to get the next source.
/// \return False if no alternative sources are available. True otherwise.
-bool PeepholeOptimizer::findNextSource(unsigned &Reg, unsigned &SubReg) {
+bool PeepholeOptimizer::findNextSource(unsigned Reg, unsigned SubReg,
+ RewriteMapTy &RewriteMap) {
// Do not try to find a new source for a physical register.
// So far we do not have any motivating example for doing that.
// Thus, instead of maintaining untested code, we will revisit that if
// that changes at some point.
if (TargetRegisterInfo::isPhysicalRegister(Reg))
return false;
-
const TargetRegisterClass *DefRC = MRI->getRegClass(Reg);
- unsigned DefSubReg = SubReg;
-
- unsigned Src;
- unsigned SrcSubReg;
- bool ShouldRewrite = false;
-
- // Follow the chain of copies until we reach the top of the use-def chain
- // or find a more suitable source.
- ValueTracker ValTracker(Reg, DefSubReg, *MRI, !DisableAdvCopyOpt, TII);
- do {
- ValueTrackerResult Res = ValTracker.getNextSource();
- if (!Res.isValid())
- break;
- Src = Res.getSrcReg(0);
- SrcSubReg = Res.getSrcSubReg(0);
-
- // Do not extend the live-ranges of physical registers as they add
- // constraints to the register allocator.
- // Moreover, if we want to extend the live-range of a physical register,
- // unlike SSA virtual register, we will have to check that they are not
- // redefine before the related use.
- if (TargetRegisterInfo::isPhysicalRegister(Src))
- break;
- const TargetRegisterClass *SrcRC = MRI->getRegClass(Src);
+ SmallVector<TargetInstrInfo::RegSubRegPair, 4> SrcToLook;
+ TargetInstrInfo::RegSubRegPair CurSrcPair(Reg, SubReg);
+ SrcToLook.push_back(CurSrcPair);
- // If this source does not incur a cross register bank copy, use it.
- ShouldRewrite = shareSameRegisterFile(*TRI, DefRC, DefSubReg, SrcRC,
- SrcSubReg);
- } while (!ShouldRewrite);
+ unsigned PHICount = 0;
+ while (!SrcToLook.empty() && PHICount < RewritePHILimit) {
+ TargetInstrInfo::RegSubRegPair Pair = SrcToLook.pop_back_val();
+ // As explained above, do not handle physical registers
+ if (TargetRegisterInfo::isPhysicalRegister(Pair.Reg))
+ return false;
- // If we did not find a more suitable source, there is nothing to optimize.
- if (!ShouldRewrite || Src == Reg)
+ CurSrcPair = Pair;
+ ValueTracker ValTracker(CurSrcPair.Reg, CurSrcPair.SubReg, *MRI,
+ !DisableAdvCopyOpt, TII);
+ ValueTrackerResult Res;
+ bool ShouldRewrite = false;
+
+ do {
+ // Follow the chain of copies until we reach the top of the use-def chain
+ // or find a more suitable source.
+ Res = ValTracker.getNextSource();
+ if (!Res.isValid())
+ break;
+
+ // Insert the Def -> Use entry for the recently found source.
+ ValueTrackerResult CurSrcRes = RewriteMap.lookup(CurSrcPair);
+ if (CurSrcRes.isValid()) {
+ assert(CurSrcRes == Res && "ValueTrackerResult found must match");
+ // An existent entry with multiple sources is a PHI cycle we must avoid.
+ // Otherwise it's an entry with a valid next source we already found.
+ if (CurSrcRes.getNumSources() > 1) {
+ DEBUG(dbgs() << "findNextSource: found PHI cycle, aborting...\n");
+ return false;
+ }
+ break;
+ }
+ RewriteMap.insert(std::make_pair(CurSrcPair, Res));
+
+ // ValueTrackerResult usually have one source unless it's the result from
+ // a PHI instruction. Add the found PHI edges to be looked up further.
+ unsigned NumSrcs = Res.getNumSources();
+ if (NumSrcs > 1) {
+ PHICount++;
+ for (unsigned i = 0; i < NumSrcs; ++i)
+ SrcToLook.push_back(TargetInstrInfo::RegSubRegPair(
+ Res.getSrcReg(i), Res.getSrcSubReg(i)));
+ break;
+ }
+
+ CurSrcPair.Reg = Res.getSrcReg(0);
+ CurSrcPair.SubReg = Res.getSrcSubReg(0);
+ // Do not extend the live-ranges of physical registers as they add
+ // constraints to the register allocator. Moreover, if we want to extend
+ // the live-range of a physical register, unlike SSA virtual register,
+ // we will have to check that they aren't redefine before the related use.
+ if (TargetRegisterInfo::isPhysicalRegister(CurSrcPair.Reg))
+ return false;
+
+ const TargetRegisterClass *SrcRC = MRI->getRegClass(CurSrcPair.Reg);
+ ShouldRewrite = TRI->shouldRewriteCopySrc(DefRC, SubReg, SrcRC,
+ CurSrcPair.SubReg);
+ } while (!ShouldRewrite);
+
+ // Continue looking for new sources...
+ if (Res.isValid())
+ continue;
+
+ // Do not continue searching for a new source if the there's at least
+ // one use-def which cannot be rewritten.
+ if (!ShouldRewrite)
+ return false;
+ }
+
+ if (PHICount >= RewritePHILimit) {
+ DEBUG(dbgs() << "findNextSource: PHI limit reached\n");
return false;
+ }
- Reg = Src;
- SubReg = SrcSubReg;
- return true;
+ // If we did not find a more suitable source, there is nothing to optimize.
+ return CurSrcPair.Reg != Reg;
+}
+
+/// \brief Insert a PHI instruction with incoming edges \p SrcRegs that are
+/// guaranteed to have the same register class. This is necessary whenever we
+/// successfully traverse a PHI instruction and find suitable sources coming
+/// from its edges. By inserting a new PHI, we provide a rewritten PHI def
+/// suitable to be used in a new COPY instruction.
+static MachineInstr *
+insertPHI(MachineRegisterInfo *MRI, const TargetInstrInfo *TII,
+ const SmallVectorImpl<TargetInstrInfo::RegSubRegPair> &SrcRegs,
+ MachineInstr *OrigPHI) {
+ assert(!SrcRegs.empty() && "No sources to create a PHI instruction?");
+
+ const TargetRegisterClass *NewRC = MRI->getRegClass(SrcRegs[0].Reg);
+ unsigned NewVR = MRI->createVirtualRegister(NewRC);
+ MachineBasicBlock *MBB = OrigPHI->getParent();
+ MachineInstrBuilder MIB = BuildMI(*MBB, OrigPHI, OrigPHI->getDebugLoc(),
+ TII->get(TargetOpcode::PHI), NewVR);
+
+ unsigned MBBOpIdx = 2;
+ for (auto RegPair : SrcRegs) {
+ MIB.addReg(RegPair.Reg, 0, RegPair.SubReg);
+ MIB.addMBB(OrigPHI->getOperand(MBBOpIdx).getMBB());
+ // Since we're extended the lifetime of RegPair.Reg, clear the
+ // kill flags to account for that and make RegPair.Reg reaches
+ // the new PHI.
+ MRI->clearKillFlags(RegPair.Reg);
+ MBBOpIdx += 2;
+ }
+
+ return MIB;
}
namespace {
/// This source defines the whole definition, i.e.,
/// (TrackReg, TrackSubReg) = (dst, dstSubIdx).
///
- /// The second and subsequent calls will return false, has there is only one
+ /// The second and subsequent calls will return false, as there is only one
/// rewritable source.
///
/// \return True if a rewritable source has been found, false otherwise.
virtual bool getNextRewritableSource(unsigned &SrcReg, unsigned &SrcSubReg,
unsigned &TrackReg,
unsigned &TrackSubReg) {
- // If CurrentSrcIdx == 1, this means this function has already been
- // called once. CopyLike has one defintiion and one argument, thus,
- // there is nothing else to rewrite.
+ // If CurrentSrcIdx == 1, this means this function has already been called
+ // once. CopyLike has one definition and one argument, thus, there is
+ // nothing else to rewrite.
if (!CopyLike.isCopy() || CurrentSrcIdx == 1)
return false;
// This is the first call to getNextRewritableSource.
return true;
}
- /// \brief Rewrite the current source with \p NewSrcReg and \p NewSecSubReg
- /// by creating a new COPY instruction. \p DefReg and \p DefSubReg contain the
- /// definition to be rewritten from the original copylike instruction.
- /// \return The new COPY if the rewriting was possible, nullptr otherwise.
- /// This is needed to handle Uncoalescable copies, since they are copy
- /// like instructions that aren't recognized by the register allocator.
- virtual MachineInstr *RewriteCurrentSource(unsigned DefReg,
- unsigned DefSubReg,
- unsigned NewSrcReg,
- unsigned NewSrcSubReg) {
+ /// \brief Given a \p Def.Reg and Def.SubReg pair, use \p RewriteMap to find
+ /// the new source to use for rewrite. If \p HandleMultipleSources is true and
+ /// multiple sources for a given \p Def are found along the way, we found a
+ /// PHI instructions that needs to be rewritten.
+ /// TODO: HandleMultipleSources should be removed once we test PHI handling
+ /// with coalescable copies.
+ TargetInstrInfo::RegSubRegPair
+ getNewSource(MachineRegisterInfo *MRI, const TargetInstrInfo *TII,
+ TargetInstrInfo::RegSubRegPair Def,
+ PeepholeOptimizer::RewriteMapTy &RewriteMap,
+ bool HandleMultipleSources = true) {
+
+ TargetInstrInfo::RegSubRegPair LookupSrc(Def.Reg, Def.SubReg);
+ do {
+ ValueTrackerResult Res = RewriteMap.lookup(LookupSrc);
+ // If there are no entries on the map, LookupSrc is the new source.
+ if (!Res.isValid())
+ return LookupSrc;
+
+ // There's only one source for this definition, keep searching...
+ unsigned NumSrcs = Res.getNumSources();
+ if (NumSrcs == 1) {
+ LookupSrc.Reg = Res.getSrcReg(0);
+ LookupSrc.SubReg = Res.getSrcSubReg(0);
+ continue;
+ }
+
+ // TODO: Remove once multiple srcs w/ coalescable copies are supported.
+ if (!HandleMultipleSources)
+ break;
+
+ // Multiple sources, recurse into each source to find a new source
+ // for it. Then, rewrite the PHI accordingly to its new edges.
+ SmallVector<TargetInstrInfo::RegSubRegPair, 4> NewPHISrcs;
+ for (unsigned i = 0; i < NumSrcs; ++i) {
+ TargetInstrInfo::RegSubRegPair PHISrc(Res.getSrcReg(i),
+ Res.getSrcSubReg(i));
+ NewPHISrcs.push_back(
+ getNewSource(MRI, TII, PHISrc, RewriteMap, HandleMultipleSources));
+ }
+
+ // Build the new PHI node and return its def register as the new source.
+ MachineInstr *OrigPHI = const_cast<MachineInstr *>(Res.getInst());
+ MachineInstr *NewPHI = insertPHI(MRI, TII, NewPHISrcs, OrigPHI);
+ DEBUG(dbgs() << "-- getNewSource\n");
+ DEBUG(dbgs() << " Replacing: " << *OrigPHI);
+ DEBUG(dbgs() << " With: " << *NewPHI);
+ const MachineOperand &MODef = NewPHI->getOperand(0);
+ return TargetInstrInfo::RegSubRegPair(MODef.getReg(), MODef.getSubReg());
+
+ } while (1);
+
+ return TargetInstrInfo::RegSubRegPair(0, 0);
+ }
+
+ /// \brief Rewrite the source found through \p Def, by using the \p RewriteMap
+ /// and create a new COPY instruction. More info about RewriteMap in
+ /// PeepholeOptimizer::findNextSource. Right now this is only used to handle
+ /// Uncoalescable copies, since they are copy like instructions that aren't
+ /// recognized by the register allocator.
+ virtual MachineInstr *
+ RewriteSource(TargetInstrInfo::RegSubRegPair Def,
+ PeepholeOptimizer::RewriteMapTy &RewriteMap) {
return nullptr;
}
};
return true;
}
- /// \brief Rewrite the current source with \p NewSrcReg and \p NewSrcSubReg
- /// by creating a new COPY instruction. \p DefReg and \p DefSubReg contain the
- /// definition to be rewritten from the original copylike instruction.
- /// \return The new COPY if the rewriting was possible, nullptr otherwise.
- MachineInstr *RewriteCurrentSource(unsigned DefReg, unsigned DefSubReg,
- unsigned NewSrcReg,
- unsigned NewSrcSubReg) override {
- assert(!TargetRegisterInfo::isPhysicalRegister(DefReg) &&
+ /// \brief Rewrite the source found through \p Def, by using the \p RewriteMap
+ /// and create a new COPY instruction. More info about RewriteMap in
+ /// PeepholeOptimizer::findNextSource. Right now this is only used to handle
+ /// Uncoalescable copies, since they are copy like instructions that aren't
+ /// recognized by the register allocator.
+ MachineInstr *
+ RewriteSource(TargetInstrInfo::RegSubRegPair Def,
+ PeepholeOptimizer::RewriteMapTy &RewriteMap) override {
+ assert(!TargetRegisterInfo::isPhysicalRegister(Def.Reg) &&
"We do not rewrite physical registers");
- const TargetRegisterClass *DefRC = MRI.getRegClass(DefReg);
+ // Find the new source to use in the COPY rewrite.
+ TargetInstrInfo::RegSubRegPair NewSrc =
+ getNewSource(&MRI, &TII, Def, RewriteMap);
+
+ // Insert the COPY.
+ const TargetRegisterClass *DefRC = MRI.getRegClass(Def.Reg);
unsigned NewVR = MRI.createVirtualRegister(DefRC);
MachineInstr *NewCopy =
BuildMI(*CopyLike.getParent(), &CopyLike, CopyLike.getDebugLoc(),
TII.get(TargetOpcode::COPY), NewVR)
- .addReg(NewSrcReg, 0, NewSrcSubReg);
+ .addReg(NewSrc.Reg, 0, NewSrc.SubReg);
- NewCopy->getOperand(0).setSubReg(DefSubReg);
- if (DefSubReg)
+ NewCopy->getOperand(0).setSubReg(Def.SubReg);
+ if (Def.SubReg)
NewCopy->getOperand(0).setIsUndef();
- MRI.replaceRegWith(DefReg, NewVR);
+ DEBUG(dbgs() << "-- RewriteSource\n");
+ DEBUG(dbgs() << " Replacing: " << CopyLike);
+ DEBUG(dbgs() << " With: " << *NewCopy);
+ MRI.replaceRegWith(Def.Reg, NewVR);
MRI.clearKillFlags(NewVR);
+ // We extended the lifetime of NewSrc.Reg, clear the kill flags to
+ // account for that.
+ MRI.clearKillFlags(NewSrc.Reg);
+
return NewCopy;
}
};
// partial definition.
TrackReg = MODef.getReg();
if (MODef.getSubReg())
- // Bails if we have to compose sub-register indices.
+ // Bail if we have to compose sub-register indices.
return false;
TrackSubReg = (unsigned)CopyLike.getOperand(3).getImm();
return true;
CurrentSrcIdx = 1;
const MachineOperand &MOExtractedReg = CopyLike.getOperand(1);
SrcReg = MOExtractedReg.getReg();
- // If we have to compose sub-register indices, bails out.
+ // If we have to compose sub-register indices, bail out.
if (MOExtractedReg.getSubReg())
return false;
}
const MachineOperand &MOInsertedReg = CopyLike.getOperand(CurrentSrcIdx);
SrcReg = MOInsertedReg.getReg();
- // If we have to compose sub-register indices, bails out.
+ // If we have to compose sub-register indices, bail out.
if ((SrcSubReg = MOInsertedReg.getSubReg()))
return false;
const MachineOperand &MODef = CopyLike.getOperand(0);
TrackReg = MODef.getReg();
- // If we have to compose sub-registers, bails.
+ // If we have to compose sub-registers, bail.
return MODef.getSubReg() == 0;
}
/// the same register bank.
/// New copies issued by this optimization are register allocator
/// friendly. This optimization does not remove any copy as it may
-/// overconstraint the register allocator, but replaces some operands
+/// overconstrain the register allocator, but replaces some operands
/// when possible.
/// \pre isCoalescableCopy(*MI) is true.
/// \return True, when \p MI has been rewritten. False otherwise.
bool Changed = false;
// Get the right rewriter for the current copy.
std::unique_ptr<CopyRewriter> CpyRewriter(getCopyRewriter(*MI, *TII, *MRI));
- // If none exists, bails out.
+ // If none exists, bail out.
if (!CpyRewriter)
return false;
// Rewrite each rewritable source.
unsigned SrcReg, SrcSubReg, TrackReg, TrackSubReg;
while (CpyRewriter->getNextRewritableSource(SrcReg, SrcSubReg, TrackReg,
TrackSubReg)) {
- unsigned NewSrc = TrackReg;
- unsigned NewSubReg = TrackSubReg;
+ // Keep track of PHI nodes and its incoming edges when looking for sources.
+ RewriteMapTy RewriteMap;
// Try to find a more suitable source. If we failed to do so, or get the
// actual source, move to the next source.
- if (!findNextSource(NewSrc, NewSubReg) || SrcReg == NewSrc)
+ if (!findNextSource(TrackReg, TrackSubReg, RewriteMap))
+ continue;
+
+ // Get the new source to rewrite. TODO: Only enable handling of multiple
+ // sources (PHIs) once we have a motivating example and testcases for it.
+ TargetInstrInfo::RegSubRegPair TrackPair(TrackReg, TrackSubReg);
+ TargetInstrInfo::RegSubRegPair NewSrc = CpyRewriter->getNewSource(
+ MRI, TII, TrackPair, RewriteMap, false /* multiple sources */);
+ if (SrcReg == NewSrc.Reg || NewSrc.Reg == 0)
continue;
+
// Rewrite source.
- if (CpyRewriter->RewriteCurrentSource(NewSrc, NewSubReg)) {
+ if (CpyRewriter->RewriteCurrentSource(NewSrc.Reg, NewSrc.SubReg)) {
// We may have extended the live-range of NewSrc, account for that.
- MRI->clearKillFlags(NewSrc);
+ MRI->clearKillFlags(NewSrc.Reg);
Changed = true;
}
}
assert(MI && isUncoalescableCopy(*MI) && "Invalid argument");
// Check if we can rewrite all the values defined by this instruction.
- SmallVector<
- std::pair<TargetInstrInfo::RegSubRegPair, TargetInstrInfo::RegSubRegPair>,
- 4> RewritePairs;
+ SmallVector<TargetInstrInfo::RegSubRegPair, 4> RewritePairs;
// Get the right rewriter for the current copy.
std::unique_ptr<CopyRewriter> CpyRewriter(getCopyRewriter(*MI, *TII, *MRI));
- // If none exists, bails out.
+ // If none exists, bail out.
if (!CpyRewriter)
return false;
// Rewrite each rewritable source by generating new COPYs. This works
// differently from optimizeCoalescableCopy since it first makes sure that all
// definitions can be rewritten.
- unsigned SrcReg, SrcSubReg, TrackReg, TrackSubReg;
- while (CpyRewriter->getNextRewritableSource(SrcReg, SrcSubReg, TrackReg,
- TrackSubReg)) {
+ RewriteMapTy RewriteMap;
+ unsigned Reg, SubReg, CopyDefReg, CopyDefSubReg;
+ while (CpyRewriter->getNextRewritableSource(Reg, SubReg, CopyDefReg,
+ CopyDefSubReg)) {
// If a physical register is here, this is probably for a good reason.
// Do not rewrite that.
- if (TargetRegisterInfo::isPhysicalRegister(TrackReg))
+ if (TargetRegisterInfo::isPhysicalRegister(CopyDefReg))
return false;
// If we do not know how to rewrite this definition, there is no point
// in trying to kill this instruction.
- TargetInstrInfo::RegSubRegPair Def(TrackReg, TrackSubReg);
- TargetInstrInfo::RegSubRegPair Src = Def;
- if (!findNextSource(Src.Reg, Src.SubReg))
+ TargetInstrInfo::RegSubRegPair Def(CopyDefReg, CopyDefSubReg);
+ if (!findNextSource(Def.Reg, Def.SubReg, RewriteMap))
return false;
- RewritePairs.push_back(std::make_pair(Def, Src));
+ RewritePairs.push_back(Def);
}
// The change is possible for all defs, do it.
- for (const auto &PairDefSrc : RewritePairs) {
- const auto &Def = PairDefSrc.first;
- const auto &Src = PairDefSrc.second;
-
+ for (const auto &Def : RewritePairs) {
// Rewrite the "copy" in a way the register coalescer understands.
- MachineInstr *NewCopy = CpyRewriter->RewriteCurrentSource(
- Def.Reg, Def.SubReg, Src.Reg, Src.SubReg);
+ MachineInstr *NewCopy = CpyRewriter->RewriteSource(Def, RewriteMap);
assert(NewCopy && "Should be able to always generate a new copy");
-
- // We extended the lifetime of Src and clear the kill flags to
- // account for that.
- MRI->clearKillFlags(Src.Reg);
LocalMIs.insert(NewCopy);
}
+
// MI is now dead.
MI->eraseFromParent();
++NumUncoalescableCopies;
return true;
}
-/// isLoadFoldable - Check whether MI is a candidate for folding into a later
-/// instruction. We only fold loads to virtual registers and the virtual
-/// register defined has a single use.
+/// Check whether MI is a candidate for folding into a later instruction.
+/// We only fold loads to virtual registers and the virtual register defined
+/// has a single use.
bool PeepholeOptimizer::isLoadFoldable(
- MachineInstr *MI,
- SmallSet<unsigned, 16> &FoldAsLoadDefCandidates) {
+ MachineInstr *MI, SmallSet<unsigned, 16> &FoldAsLoadDefCandidates) {
if (!MI->canFoldAsLoad() || !MI->mayLoad())
return false;
const MCInstrDesc &MCID = MI->getDesc();
return false;
}
-bool PeepholeOptimizer::isMoveImmediate(MachineInstr *MI,
- SmallSet<unsigned, 4> &ImmDefRegs,
- DenseMap<unsigned, MachineInstr*> &ImmDefMIs) {
+bool PeepholeOptimizer::isMoveImmediate(
+ MachineInstr *MI, SmallSet<unsigned, 4> &ImmDefRegs,
+ DenseMap<unsigned, MachineInstr *> &ImmDefMIs) {
const MCInstrDesc &MCID = MI->getDesc();
if (!MI->isMoveImmediate())
return false;
return false;
}
-/// foldImmediate - Try folding register operands that are defined by move
-/// immediate instructions, i.e. a trivial constant folding optimization, if
+/// Try folding register operands that are defined by move immediate
+/// instructions, i.e. a trivial constant folding optimization, if
/// and only if the def and use are in the same BB.
-bool PeepholeOptimizer::foldImmediate(MachineInstr *MI, MachineBasicBlock *MBB,
- SmallSet<unsigned, 4> &ImmDefRegs,
- DenseMap<unsigned, MachineInstr*> &ImmDefMIs) {
+bool PeepholeOptimizer::foldImmediate(
+ MachineInstr *MI, MachineBasicBlock *MBB, SmallSet<unsigned, 4> &ImmDefRegs,
+ DenseMap<unsigned, MachineInstr *> &ImmDefMIs) {
for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg() || MO.isDef())
continue;
+ // Ignore dead implicit defs.
+ if (MO.isImplicit() && MO.isDead())
+ continue;
unsigned Reg = MO.getReg();
if (!TargetRegisterInfo::isVirtualRegister(Reg))
continue;
if (ImmDefRegs.count(Reg) == 0)
continue;
DenseMap<unsigned, MachineInstr*>::iterator II = ImmDefMIs.find(Reg);
- assert(II != ImmDefMIs.end());
+ assert(II != ImmDefMIs.end() && "couldn't find immediate definition");
if (TII->FoldImmediate(MI, II->second, Reg, MRI)) {
++NumImmFold;
return true;
return false;
}
+// FIXME: This is very simple and misses some cases which should be handled when
+// motivating examples are found.
+//
+// The copy rewriting logic should look at uses as well as defs and be able to
+// eliminate copies across blocks.
+//
+// Later copies that are subregister extracts will also not be eliminated since
+// only the first copy is considered.
+//
+// e.g.
+// %vreg1 = COPY %vreg0
+// %vreg2 = COPY %vreg0:sub1
+//
+// Should replace %vreg2 uses with %vreg1:sub1
+bool PeepholeOptimizer::foldRedundantCopy(
+ MachineInstr *MI, SmallSet<unsigned, 4> &CopySrcRegs,
+ DenseMap<unsigned, MachineInstr *> &CopyMIs) {
+ assert(MI->isCopy() && "expected a COPY machine instruction");
+
+ unsigned SrcReg = MI->getOperand(1).getReg();
+ if (!TargetRegisterInfo::isVirtualRegister(SrcReg))
+ return false;
+
+ unsigned DstReg = MI->getOperand(0).getReg();
+ if (!TargetRegisterInfo::isVirtualRegister(DstReg))
+ return false;
+
+ if (CopySrcRegs.insert(SrcReg).second) {
+ // First copy of this reg seen.
+ CopyMIs.insert(std::make_pair(SrcReg, MI));
+ return false;
+ }
+
+ MachineInstr *PrevCopy = CopyMIs.find(SrcReg)->second;
+
+ unsigned SrcSubReg = MI->getOperand(1).getSubReg();
+ unsigned PrevSrcSubReg = PrevCopy->getOperand(1).getSubReg();
+
+ // Can't replace different subregister extracts.
+ if (SrcSubReg != PrevSrcSubReg)
+ return false;
+
+ unsigned PrevDstReg = PrevCopy->getOperand(0).getReg();
+
+ // Only replace if the copy register class is the same.
+ //
+ // TODO: If we have multiple copies to different register classes, we may want
+ // to track multiple copies of the same source register.
+ if (MRI->getRegClass(DstReg) != MRI->getRegClass(PrevDstReg))
+ return false;
+
+ MRI->replaceRegWith(DstReg, PrevDstReg);
+
+ // Lifetime of the previous copy has been extended.
+ MRI->clearKillFlags(PrevDstReg);
+ return true;
+}
+
+bool PeepholeOptimizer::isNAPhysCopy(unsigned Reg) {
+ return TargetRegisterInfo::isPhysicalRegister(Reg) &&
+ !MRI->isAllocatable(Reg);
+}
+
+bool PeepholeOptimizer::foldRedundantNAPhysCopy(
+ MachineInstr *MI, DenseMap<unsigned, MachineInstr *> &NAPhysToVirtMIs) {
+ assert(MI->isCopy() && "expected a COPY machine instruction");
+
+ if (DisableNAPhysCopyOpt)
+ return false;
+
+ unsigned DstReg = MI->getOperand(0).getReg();
+ unsigned SrcReg = MI->getOperand(1).getReg();
+ if (isNAPhysCopy(SrcReg) && TargetRegisterInfo::isVirtualRegister(DstReg)) {
+ // %vreg = COPY %PHYSREG
+ // Avoid using a datastructure which can track multiple live non-allocatable
+ // phys->virt copies since LLVM doesn't seem to do this.
+ NAPhysToVirtMIs.insert({SrcReg, MI});
+ return false;
+ }
+
+ if (!(TargetRegisterInfo::isVirtualRegister(SrcReg) && isNAPhysCopy(DstReg)))
+ return false;
+
+ // %PHYSREG = COPY %vreg
+ auto PrevCopy = NAPhysToVirtMIs.find(DstReg);
+ if (PrevCopy == NAPhysToVirtMIs.end()) {
+ // We can't remove the copy: there was an intervening clobber of the
+ // non-allocatable physical register after the copy to virtual.
+ DEBUG(dbgs() << "NAPhysCopy: intervening clobber forbids erasing " << *MI
+ << '\n');
+ return false;
+ }
+
+ unsigned PrevDstReg = PrevCopy->second->getOperand(0).getReg();
+ if (PrevDstReg == SrcReg) {
+ // Remove the virt->phys copy: we saw the virtual register definition, and
+ // the non-allocatable physical register's state hasn't changed since then.
+ DEBUG(dbgs() << "NAPhysCopy: erasing " << *MI << '\n');
+ ++NumNAPhysCopies;
+ return true;
+ }
+
+ // Potential missed optimization opportunity: we saw a different virtual
+ // register get a copy of the non-allocatable physical register, and we only
+ // track one such copy. Avoid getting confused by this new non-allocatable
+ // physical register definition, and remove it from the tracked copies.
+ DEBUG(dbgs() << "NAPhysCopy: missed opportunity " << *MI << '\n');
+ NAPhysToVirtMIs.erase(PrevCopy);
+ return false;
+}
+
bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) {
if (skipOptnoneFunction(*MF.getFunction()))
return false;
bool Changed = false;
- for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
- MachineBasicBlock *MBB = &*I;
-
+ for (MachineBasicBlock &MBB : MF) {
bool SeenMoveImm = false;
// During this forward scan, at some point it needs to answer the question
DenseMap<unsigned, MachineInstr*> ImmDefMIs;
SmallSet<unsigned, 16> FoldAsLoadDefCandidates;
- for (MachineBasicBlock::iterator
- MII = I->begin(), MIE = I->end(); MII != MIE; ) {
+ // Track when a non-allocatable physical register is copied to a virtual
+ // register so that useless moves can be removed.
+ //
+ // %PHYSREG is the map index; MI is the last valid `%vreg = COPY %PHYSREG`
+ // without any intervening re-definition of %PHYSREG.
+ DenseMap<unsigned, MachineInstr *> NAPhysToVirtMIs;
+
+ // Set of virtual registers that are copied from.
+ SmallSet<unsigned, 4> CopySrcRegs;
+ DenseMap<unsigned, MachineInstr *> CopySrcMIs;
+
+ for (MachineBasicBlock::iterator MII = MBB.begin(), MIE = MBB.end();
+ MII != MIE; ) {
MachineInstr *MI = &*MII;
// We may be erasing MI below, increment MII now.
++MII;
if (MI->isDebugValue())
continue;
- // If there exists an instruction which belongs to the following
- // categories, we will discard the load candidates.
- if (MI->isPosition() || MI->isPHI() || MI->isImplicitDef() ||
- MI->isKill() || MI->isInlineAsm() ||
- MI->hasUnmodeledSideEffects()) {
+ // If we run into an instruction we can't fold across, discard
+ // the load candidates.
+ if (MI->isLoadFoldBarrier())
FoldAsLoadDefCandidates.clear();
+
+ if (MI->isPosition() || MI->isPHI())
+ continue;
+
+ if (!MI->isCopy()) {
+ for (const auto &Op : MI->operands()) {
+ // Visit all operands: definitions can be implicit or explicit.
+ if (Op.isReg()) {
+ unsigned Reg = Op.getReg();
+ if (Op.isDef() && isNAPhysCopy(Reg)) {
+ const auto &Def = NAPhysToVirtMIs.find(Reg);
+ if (Def != NAPhysToVirtMIs.end()) {
+ // A new definition of the non-allocatable physical register
+ // invalidates previous copies.
+ DEBUG(dbgs() << "NAPhysCopy: invalidating because of " << *MI
+ << '\n');
+ NAPhysToVirtMIs.erase(Def);
+ }
+ }
+ } else if (Op.isRegMask()) {
+ const uint32_t *RegMask = Op.getRegMask();
+ for (auto &RegMI : NAPhysToVirtMIs) {
+ unsigned Def = RegMI.first;
+ if (MachineOperand::clobbersPhysReg(RegMask, Def)) {
+ DEBUG(dbgs() << "NAPhysCopy: invalidating because of " << *MI
+ << '\n');
+ NAPhysToVirtMIs.erase(Def);
+ }
+ }
+ }
+ }
+ }
+
+ if (MI->isImplicitDef() || MI->isKill())
+ continue;
+
+ if (MI->isInlineAsm() || MI->hasUnmodeledSideEffects()) {
+ // Blow away all non-allocatable physical registers knowledge since we
+ // don't know what's correct anymore.
+ //
+ // FIXME: handle explicit asm clobbers.
+ DEBUG(dbgs() << "NAPhysCopy: blowing away all info due to " << *MI
+ << '\n');
+ NAPhysToVirtMIs.clear();
continue;
}
- if (MI->mayStore() || MI->isCall())
- FoldAsLoadDefCandidates.clear();
if ((isUncoalescableCopy(*MI) &&
optimizeUncoalescableCopy(MI, LocalMIs)) ||
- (MI->isCompare() && optimizeCmpInstr(MI, MBB)) ||
+ (MI->isCompare() && optimizeCmpInstr(MI, &MBB)) ||
(MI->isSelect() && optimizeSelect(MI, LocalMIs))) {
// MI is deleted.
LocalMIs.erase(MI);
continue;
}
+ if (MI->isCopy() &&
+ (foldRedundantCopy(MI, CopySrcRegs, CopySrcMIs) ||
+ foldRedundantNAPhysCopy(MI, NAPhysToVirtMIs))) {
+ LocalMIs.erase(MI);
+ MI->eraseFromParent();
+ Changed = true;
+ continue;
+ }
+
if (isMoveImmediate(MI, ImmDefRegs, ImmDefMIs)) {
SeenMoveImm = true;
} else {
- Changed |= optimizeExtInstr(MI, MBB, LocalMIs);
+ Changed |= optimizeExtInstr(MI, &MBB, LocalMIs);
// optimizeExtInstr might have created new instructions after MI
// and before the already incremented MII. Adjust MII so that the
// next iteration sees the new instructions.
MII = MI;
++MII;
if (SeenMoveImm)
- Changed |= foldImmediate(MI, MBB, ImmDefRegs, ImmDefMIs);
+ Changed |= foldImmediate(MI, &MBB, ImmDefRegs, ImmDefMIs);
}
// Check whether MI is a load candidate for folding into a later
if (Def->getOperand(DefIdx).getSubReg() != DefSubReg)
// If we look for a different subreg, it means we want a subreg of src.
- // Bails as we do not support composing subreg yet.
+ // Bails as we do not support composing subregs yet.
return ValueTrackerResult();
// Otherwise, we want the whole source.
const MachineOperand &Src = Def->getOperand(1);
return ValueTrackerResult();
if (Def->getOperand(DefIdx).getSubReg() != DefSubReg)
// If we look for a different subreg, it means we want a subreg of the src.
- // Bails as we do not support composing subreg yet.
+ // Bails as we do not support composing subregs yet.
return ValueTrackerResult();
unsigned SrcIdx = Def->getNumOperands();
const MachineOperand &MO = Def->getOperand(OpIdx);
if (!MO.isReg() || !MO.getReg())
continue;
+ // Ignore dead implicit defs.
+ if (MO.isImplicit() && MO.isDead())
+ continue;
assert(!MO.isDef() && "We should have skipped all the definitions by now");
if (SrcIdx != EndOpIdx)
// Multiple sources?
"Invalid definition");
if (Def->getOperand(DefIdx).getSubReg())
- // If we are composing subreg, bails out.
+ // If we are composing subregs, bail out.
// The case we are checking is Def.<subreg> = REG_SEQUENCE.
// This should almost never happen as the SSA property is tracked at
// the register level (as opposed to the subreg level).
for (auto &RegSeqInput : RegSeqInputRegs) {
if (RegSeqInput.SubIdx == DefSubReg) {
if (RegSeqInput.SubReg)
- // Bails if we have to compose sub registers.
+ // Bail if we have to compose sub registers.
return ValueTrackerResult();
return ValueTrackerResult(RegSeqInput.Reg, RegSeqInput.SubReg);
"Invalid definition");
if (Def->getOperand(DefIdx).getSubReg())
- // If we are composing subreg, bails out.
+ // If we are composing subreg, bail out.
// Same remark as getNextSourceFromRegSequence.
// I.e., this may be turned into an assert.
return ValueTrackerResult();
const MachineOperand &MODef = Def->getOperand(DefIdx);
// If the result register (Def) and the base register (v0) do not
// have the same register class or if we have to compose
- // subregisters, bails out.
+ // subregisters, bail out.
if (MRI.getRegClass(MODef.getReg()) != MRI.getRegClass(BaseReg.Reg) ||
BaseReg.SubReg)
return ValueTrackerResult();
// We are looking at:
// Def = EXTRACT_SUBREG v0, sub0
- // Bails if we have to compose sub registers.
+ // Bail if we have to compose sub registers.
// Indeed, if DefSubReg != 0, we would have to compose it with sub0.
if (DefSubReg)
return ValueTrackerResult();
if (!TII->getExtractSubregInputs(*Def, DefIdx, ExtractSubregInputReg))
return ValueTrackerResult();
- // Bails if we have to compose sub registers.
+ // Bail if we have to compose sub registers.
// Likewise, if v0.subreg != 0, we would have to compose v0.subreg with sub0.
if (ExtractSubregInputReg.SubReg)
return ValueTrackerResult();
// Otherwise, the value is available in the v0.sub0.
- return ValueTrackerResult(ExtractSubregInputReg.Reg, ExtractSubregInputReg.SubIdx);
+ return ValueTrackerResult(ExtractSubregInputReg.Reg,
+ ExtractSubregInputReg.SubIdx);
}
ValueTrackerResult ValueTracker::getNextSourceFromSubregToReg() {
// We are looking at:
// Def = SUBREG_TO_REG Imm, v0, sub0
- // Bails if we have to compose sub registers.
+ // Bail if we have to compose sub registers.
// If DefSubReg != sub0, we would have to check that all the bits
// we track are included in sub0 and if yes, we would have to
// determine the right subreg in v0.
if (DefSubReg != Def->getOperand(3).getImm())
return ValueTrackerResult();
- // Bails if we have to compose sub registers.
+ // Bail if we have to compose sub registers.
// Likewise, if v0.subreg != 0, we would have to compose it with sub0.
if (Def->getOperand(2).getSubReg())
return ValueTrackerResult();
Def->getOperand(3).getImm());
}
+/// \brief Explore each PHI incoming operand and return its sources
+ValueTrackerResult ValueTracker::getNextSourceFromPHI() {
+ assert(Def->isPHI() && "Invalid definition");
+ ValueTrackerResult Res;
+
+ // If we look for a different subreg, bail as we do not support composing
+ // subregs yet.
+ if (Def->getOperand(0).getSubReg() != DefSubReg)
+ return ValueTrackerResult();
+
+ // Return all register sources for PHI instructions.
+ for (unsigned i = 1, e = Def->getNumOperands(); i < e; i += 2) {
+ auto &MO = Def->getOperand(i);
+ assert(MO.isReg() && "Invalid PHI instruction");
+ Res.addSource(MO.getReg(), MO.getSubReg());
+ }
+
+ return Res;
+}
+
ValueTrackerResult ValueTracker::getNextSourceImpl() {
assert(Def && "This method needs a valid definition");
if (Def->isBitcast())
return getNextSourceFromBitcast();
// All the remaining cases involve "complex" instructions.
- // Bails if we did not ask for the advanced tracking.
+ // Bail if we did not ask for the advanced tracking.
if (!UseAdvancedTracking)
return ValueTrackerResult();
if (Def->isRegSequence() || Def->isRegSequenceLike())
return getNextSourceFromExtractSubreg();
if (Def->isSubregToReg())
return getNextSourceFromSubregToReg();
+ if (Def->isPHI())
+ return getNextSourceFromPHI();
return ValueTrackerResult();
}