1 //===-- MachineSink.cpp - Sinking for machine instructions ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass moves instructions into successor blocks when possible, so that
11 // they aren't executed on paths where their results aren't needed.
13 // This pass is not intended to be a replacement or a complete alternative
14 // for an LLVM-IR-level sinking pass. It is only designed to sink simple
15 // constructs that are not exposed before lowering and instruction selection.
17 //===----------------------------------------------------------------------===//
19 #define DEBUG_TYPE "machine-sink"
20 #include "llvm/CodeGen/Passes.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/MachineDominators.h"
23 #include "llvm/CodeGen/MachineLoopInfo.h"
24 #include "llvm/Analysis/AliasAnalysis.h"
25 #include "llvm/Target/TargetRegisterInfo.h"
26 #include "llvm/Target/TargetInstrInfo.h"
27 #include "llvm/Target/TargetMachine.h"
28 #include "llvm/ADT/SmallSet.h"
29 #include "llvm/ADT/Statistic.h"
30 #include "llvm/Support/CommandLine.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/raw_ostream.h"
36 SplitEdges("machine-sink-split",
37 cl::desc("Split critical edges during machine sinking"),
38 cl::init(true), cl::Hidden);
40 STATISTIC(NumSunk, "Number of machine instructions sunk");
41 STATISTIC(NumSplit, "Number of critical edges split");
42 STATISTIC(NumCoalesces, "Number of copies coalesced");
45 class MachineSinking : public MachineFunctionPass {
46 const TargetInstrInfo *TII;
47 const TargetRegisterInfo *TRI;
48 MachineRegisterInfo *MRI; // Machine register information
49 MachineDominatorTree *DT; // Machine dominator tree
52 BitVector AllocatableSet; // Which physregs are allocatable?
54 // Remember which edges have been considered for breaking.
55 SmallSet<std::pair<MachineBasicBlock*,MachineBasicBlock*>, 8>
59 static char ID; // Pass identification
60 MachineSinking() : MachineFunctionPass(ID) {}
62 virtual bool runOnMachineFunction(MachineFunction &MF);
64 virtual void getAnalysisUsage(AnalysisUsage &AU) const {
66 MachineFunctionPass::getAnalysisUsage(AU);
67 AU.addRequired<AliasAnalysis>();
68 AU.addRequired<MachineDominatorTree>();
69 AU.addRequired<MachineLoopInfo>();
70 AU.addPreserved<MachineDominatorTree>();
71 AU.addPreserved<MachineLoopInfo>();
74 virtual void releaseMemory() {
75 CEBCandidates.clear();
79 bool ProcessBlock(MachineBasicBlock &MBB);
80 bool isWorthBreakingCriticalEdge(MachineInstr *MI,
81 MachineBasicBlock *From,
82 MachineBasicBlock *To);
83 MachineBasicBlock *SplitCriticalEdge(MachineInstr *MI,
84 MachineBasicBlock *From,
85 MachineBasicBlock *To,
87 bool SinkInstruction(MachineInstr *MI, bool &SawStore);
88 bool AllUsesDominatedByBlock(unsigned Reg, MachineBasicBlock *MBB,
89 MachineBasicBlock *DefMBB,
90 bool &BreakPHIEdge, bool &LocalUse) const;
91 bool PerformTrivialForwardCoalescing(MachineInstr *MI,
92 MachineBasicBlock *MBB);
94 } // end anonymous namespace
96 char MachineSinking::ID = 0;
97 INITIALIZE_PASS(MachineSinking, "machine-sink",
98 "Machine code sinking", false, false);
100 FunctionPass *llvm::createMachineSinkingPass() { return new MachineSinking(); }
102 bool MachineSinking::PerformTrivialForwardCoalescing(MachineInstr *MI,
103 MachineBasicBlock *MBB) {
107 unsigned SrcReg = MI->getOperand(1).getReg();
108 unsigned DstReg = MI->getOperand(0).getReg();
109 if (!TargetRegisterInfo::isVirtualRegister(SrcReg) ||
110 !TargetRegisterInfo::isVirtualRegister(DstReg) ||
111 !MRI->hasOneNonDBGUse(SrcReg))
114 const TargetRegisterClass *SRC = MRI->getRegClass(SrcReg);
115 const TargetRegisterClass *DRC = MRI->getRegClass(DstReg);
119 MachineInstr *DefMI = MRI->getVRegDef(SrcReg);
120 if (DefMI->isCopyLike())
122 DEBUG(dbgs() << "Coalescing: " << *DefMI);
123 DEBUG(dbgs() << "*** to: " << *MI);
124 MRI->replaceRegWith(DstReg, SrcReg);
125 MI->eraseFromParent();
130 /// AllUsesDominatedByBlock - Return true if all uses of the specified register
131 /// occur in blocks dominated by the specified block. If any use is in the
132 /// definition block, then return false since it is never legal to move def
135 MachineSinking::AllUsesDominatedByBlock(unsigned Reg,
136 MachineBasicBlock *MBB,
137 MachineBasicBlock *DefMBB,
139 bool &LocalUse) const {
140 assert(TargetRegisterInfo::isVirtualRegister(Reg) &&
141 "Only makes sense for vregs");
143 if (MRI->use_nodbg_empty(Reg))
146 // Ignoring debug uses is necessary so debug info doesn't affect the code.
147 // This may leave a referencing dbg_value in the original block, before
148 // the definition of the vreg. Dwarf generator handles this although the
149 // user might not get the right info at runtime.
151 // BreakPHIEdge is true if all the uses are in the successor MBB being sunken
152 // into and they are all PHI nodes. In this case, machine-sink must break
153 // the critical edge first. e.g.
155 // BB#1: derived from LLVM BB %bb4.preheader
156 // Predecessors according to CFG: BB#0
158 // %reg16385<def> = DEC64_32r %reg16437, %EFLAGS<imp-def,dead>
160 // JE_4 <BB#37>, %EFLAGS<imp-use>
161 // Successors according to CFG: BB#37 BB#2
163 // BB#2: derived from LLVM BB %bb.nph
164 // Predecessors according to CFG: BB#0 BB#1
165 // %reg16386<def> = PHI %reg16434, <BB#0>, %reg16385, <BB#1>
167 for (MachineRegisterInfo::use_nodbg_iterator
168 I = MRI->use_nodbg_begin(Reg), E = MRI->use_nodbg_end();
170 MachineInstr *UseInst = &*I;
171 MachineBasicBlock *UseBlock = UseInst->getParent();
172 if (!(UseBlock == MBB && UseInst->isPHI() &&
173 UseInst->getOperand(I.getOperandNo()+1).getMBB() == DefMBB)) {
174 BreakPHIEdge = false;
181 for (MachineRegisterInfo::use_nodbg_iterator
182 I = MRI->use_nodbg_begin(Reg), E = MRI->use_nodbg_end();
184 // Determine the block of the use.
185 MachineInstr *UseInst = &*I;
186 MachineBasicBlock *UseBlock = UseInst->getParent();
187 if (UseInst->isPHI()) {
188 // PHI nodes use the operand in the predecessor block, not the block with
190 UseBlock = UseInst->getOperand(I.getOperandNo()+1).getMBB();
191 } else if (UseBlock == DefMBB) {
196 // Check that it dominates.
197 if (!DT->dominates(MBB, UseBlock))
204 bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
205 DEBUG(dbgs() << "******** Machine Sinking ********\n");
207 const TargetMachine &TM = MF.getTarget();
208 TII = TM.getInstrInfo();
209 TRI = TM.getRegisterInfo();
210 MRI = &MF.getRegInfo();
211 DT = &getAnalysis<MachineDominatorTree>();
212 LI = &getAnalysis<MachineLoopInfo>();
213 AA = &getAnalysis<AliasAnalysis>();
214 AllocatableSet = TRI->getAllocatableSet(MF);
216 bool EverMadeChange = false;
219 bool MadeChange = false;
221 // Process all basic blocks.
222 CEBCandidates.clear();
223 for (MachineFunction::iterator I = MF.begin(), E = MF.end();
225 MadeChange |= ProcessBlock(*I);
227 // If this iteration over the code changed anything, keep iterating.
228 if (!MadeChange) break;
229 EverMadeChange = true;
231 return EverMadeChange;
234 bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) {
235 // Can't sink anything out of a block that has less than two successors.
236 if (MBB.succ_size() <= 1 || MBB.empty()) return false;
238 // Don't bother sinking code out of unreachable blocks. In addition to being
239 // unprofitable, it can also lead to infinite looping, because in an
240 // unreachable loop there may be nowhere to stop.
241 if (!DT->isReachableFromEntry(&MBB)) return false;
243 bool MadeChange = false;
245 // Walk the basic block bottom-up. Remember if we saw a store.
246 MachineBasicBlock::iterator I = MBB.end();
248 bool ProcessedBegin, SawStore = false;
250 MachineInstr *MI = I; // The instruction to sink.
252 // Predecrement I (if it's not begin) so that it isn't invalidated by
254 ProcessedBegin = I == MBB.begin();
258 if (MI->isDebugValue())
261 if (PerformTrivialForwardCoalescing(MI, &MBB))
264 if (SinkInstruction(MI, SawStore))
265 ++NumSunk, MadeChange = true;
267 // If we just processed the first instruction in the block, we're done.
268 } while (!ProcessedBegin);
273 bool MachineSinking::isWorthBreakingCriticalEdge(MachineInstr *MI,
274 MachineBasicBlock *From,
275 MachineBasicBlock *To) {
276 // FIXME: Need much better heuristics.
278 // If the pass has already considered breaking this edge (during this pass
279 // through the function), then let's go ahead and break it. This means
280 // sinking multiple "cheap" instructions into the same block.
281 if (!CEBCandidates.insert(std::make_pair(From, To)))
284 if (!MI->isCopy() && !MI->getDesc().isAsCheapAsAMove())
287 // MI is cheap, we probably don't want to break the critical edge for it.
288 // However, if this would allow some definitions of its source operands
289 // to be sunk then it's probably worth it.
290 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
291 const MachineOperand &MO = MI->getOperand(i);
292 if (!MO.isReg()) continue;
293 unsigned Reg = MO.getReg();
294 if (Reg == 0 || !TargetRegisterInfo::isPhysicalRegister(Reg))
296 if (MRI->hasOneNonDBGUse(Reg))
303 MachineBasicBlock *MachineSinking::SplitCriticalEdge(MachineInstr *MI,
304 MachineBasicBlock *FromBB,
305 MachineBasicBlock *ToBB,
307 if (!isWorthBreakingCriticalEdge(MI, FromBB, ToBB))
310 // Avoid breaking back edge. From == To means backedge for single BB loop.
311 if (!SplitEdges || FromBB == ToBB)
314 // Check for backedges of more "complex" loops.
315 if (LI->getLoopFor(FromBB) == LI->getLoopFor(ToBB) &&
316 LI->isLoopHeader(ToBB))
319 // It's not always legal to break critical edges and sink the computation
327 // ... no uses of v1024
333 // If BB#1 -> BB#3 edge is broken and computation of v1024 is inserted:
342 // ... no uses of v1024
348 // This is incorrect since v1024 is not computed along the BB#1->BB#2->BB#3
349 // flow. We need to ensure the new basic block where the computation is
350 // sunk to dominates all the uses.
351 // It's only legal to break critical edge and sink the computation to the
352 // new block if all the predecessors of "To", except for "From", are
353 // not dominated by "From". Given SSA property, this means these
354 // predecessors are dominated by "To".
356 // There is no need to do this check if all the uses are PHI nodes. PHI
357 // sources are only defined on the specific predecessor edges.
359 for (MachineBasicBlock::pred_iterator PI = ToBB->pred_begin(),
360 E = ToBB->pred_end(); PI != E; ++PI) {
363 if (!DT->dominates(ToBB, *PI))
368 return FromBB->SplitCriticalEdge(ToBB, this);
371 static bool AvoidsSinking(MachineInstr *MI, MachineRegisterInfo *MRI) {
372 return MI->isInsertSubreg() || MI->isSubregToReg() || MI->isRegSequence();
375 /// SinkInstruction - Determine whether it is safe to sink the specified machine
376 /// instruction out of its current block into a successor.
377 bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
378 // Don't sink insert_subreg, subreg_to_reg, reg_sequence. These are meant to
379 // be close to the source to make it easier to coalesce.
380 if (AvoidsSinking(MI, MRI))
383 // Check if it's safe to move the instruction.
384 if (!MI->isSafeToMove(TII, AA, SawStore))
387 // FIXME: This should include support for sinking instructions within the
388 // block they are currently in to shorten the live ranges. We often get
389 // instructions sunk into the top of a large block, but it would be better to
390 // also sink them down before their first use in the block. This xform has to
391 // be careful not to *increase* register pressure though, e.g. sinking
392 // "x = y + z" down if it kills y and z would increase the live ranges of y
393 // and z and only shrink the live range of x.
395 // Loop over all the operands of the specified instruction. If there is
396 // anything we can't handle, bail out.
397 MachineBasicBlock *ParentBlock = MI->getParent();
399 // SuccToSinkTo - This is the successor to sink this instruction to, once we
401 MachineBasicBlock *SuccToSinkTo = 0;
403 bool BreakPHIEdge = false;
404 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
405 const MachineOperand &MO = MI->getOperand(i);
406 if (!MO.isReg()) continue; // Ignore non-register operands.
408 unsigned Reg = MO.getReg();
409 if (Reg == 0) continue;
411 if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
413 // If the physreg has no defs anywhere, it's just an ambient register
414 // and we can freely move its uses. Alternatively, if it's allocatable,
415 // it could get allocated to something with a def during allocation.
416 if (!MRI->def_empty(Reg))
419 if (AllocatableSet.test(Reg))
422 // Check for a def among the register's aliases too.
423 for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) {
424 unsigned AliasReg = *Alias;
425 if (!MRI->def_empty(AliasReg))
428 if (AllocatableSet.test(AliasReg))
431 } else if (!MO.isDead()) {
432 // A def that isn't dead. We can't move it.
436 // Virtual register uses are always safe to sink.
437 if (MO.isUse()) continue;
439 // If it's not safe to move defs of the register class, then abort.
440 if (!TII->isSafeToMoveRegClassDefs(MRI->getRegClass(Reg)))
443 // FIXME: This picks a successor to sink into based on having one
444 // successor that dominates all the uses. However, there are cases where
445 // sinking can happen but where the sink point isn't a successor. For
452 // the instruction could be sunk over the whole diamond for the
453 // if/then/else (or loop, etc), allowing it to be sunk into other blocks
456 // Virtual register defs can only be sunk if all their uses are in blocks
457 // dominated by one of the successors.
459 // If a previous operand picked a block to sink to, then this operand
460 // must be sinkable to the same block.
461 bool LocalUse = false;
462 if (!AllUsesDominatedByBlock(Reg, SuccToSinkTo, ParentBlock,
463 BreakPHIEdge, LocalUse))
469 // Otherwise, we should look at all the successors and decide which one
470 // we should sink to.
471 for (MachineBasicBlock::succ_iterator SI = ParentBlock->succ_begin(),
472 E = ParentBlock->succ_end(); SI != E; ++SI) {
473 bool LocalUse = false;
474 if (AllUsesDominatedByBlock(Reg, *SI, ParentBlock,
475 BreakPHIEdge, LocalUse)) {
480 // Def is used locally, it's never safe to move this def.
484 // If we couldn't find a block to sink to, ignore this instruction.
485 if (SuccToSinkTo == 0)
490 // If there are no outputs, it must have side-effects.
491 if (SuccToSinkTo == 0)
494 // It's not safe to sink instructions to EH landing pad. Control flow into
495 // landing pad is implicitly defined.
496 if (SuccToSinkTo->isLandingPad())
499 // It is not possible to sink an instruction into its own block. This can
500 // happen with loops.
501 if (MI->getParent() == SuccToSinkTo)
504 // If the instruction to move defines a dead physical register which is live
505 // when leaving the basic block, don't move it because it could turn into a
506 // "zombie" define of that preg. E.g., EFLAGS. (<rdar://problem/8030636>)
507 for (unsigned I = 0, E = MI->getNumOperands(); I != E; ++I) {
508 const MachineOperand &MO = MI->getOperand(I);
509 if (!MO.isReg()) continue;
510 unsigned Reg = MO.getReg();
511 if (Reg == 0 || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue;
512 if (SuccToSinkTo->isLiveIn(Reg))
516 DEBUG(dbgs() << "Sink instr " << *MI << "\tinto block " << *SuccToSinkTo);
518 // If the block has multiple predecessors, this would introduce computation on
519 // a path that it doesn't already exist. We could split the critical edge,
520 // but for now we just punt.
521 if (SuccToSinkTo->pred_size() > 1) {
522 // We cannot sink a load across a critical edge - there may be stores in
524 bool TryBreak = false;
526 if (!MI->isSafeToMove(TII, AA, store)) {
527 DEBUG(dbgs() << " *** NOTE: Won't sink load along critical edge.\n");
531 // We don't want to sink across a critical edge if we don't dominate the
532 // successor. We could be introducing calculations to new code paths.
533 if (!TryBreak && !DT->dominates(ParentBlock, SuccToSinkTo)) {
534 DEBUG(dbgs() << " *** NOTE: Critical edge found\n");
538 // Don't sink instructions into a loop.
539 if (!TryBreak && LI->isLoopHeader(SuccToSinkTo)) {
540 DEBUG(dbgs() << " *** NOTE: Loop header found\n");
544 // Otherwise we are OK with sinking along a critical edge.
546 DEBUG(dbgs() << "Sinking along critical edge.\n");
548 MachineBasicBlock *NewSucc =
549 SplitCriticalEdge(MI, ParentBlock, SuccToSinkTo, BreakPHIEdge);
551 DEBUG(dbgs() << " *** PUNTING: Not legal or profitable to "
552 "break critical edge\n");
555 DEBUG(dbgs() << " *** Splitting critical edge:"
556 " BB#" << ParentBlock->getNumber()
557 << " -- BB#" << NewSucc->getNumber()
558 << " -- BB#" << SuccToSinkTo->getNumber() << '\n');
559 SuccToSinkTo = NewSucc;
561 BreakPHIEdge = false;
567 // BreakPHIEdge is true if all the uses are in the successor MBB being
568 // sunken into and they are all PHI nodes. In this case, machine-sink must
569 // break the critical edge first.
570 MachineBasicBlock *NewSucc = SplitCriticalEdge(MI, ParentBlock,
571 SuccToSinkTo, BreakPHIEdge);
573 DEBUG(dbgs() << " *** PUNTING: Not legal or profitable to "
574 "break critical edge\n");
578 DEBUG(dbgs() << " *** Splitting critical edge:"
579 " BB#" << ParentBlock->getNumber()
580 << " -- BB#" << NewSucc->getNumber()
581 << " -- BB#" << SuccToSinkTo->getNumber() << '\n');
582 SuccToSinkTo = NewSucc;
586 // Determine where to insert into. Skip phi nodes.
587 MachineBasicBlock::iterator InsertPos = SuccToSinkTo->begin();
588 while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
591 // Move the instruction.
592 SuccToSinkTo->splice(InsertPos, ParentBlock, MI,
593 ++MachineBasicBlock::iterator(MI));
595 // Conservatively, clear any kill flags, since it's possible that they are no