lib/Target/R600/R600ControlFlowFinalizer.cpp

   1 //===-- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst----------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 /// \file
  11 /// This pass compute turns all control flow pseudo instructions into native one
  12 /// computing their address on the fly ; it also sets STACK_SIZE info.
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "llvm/Support/Debug.h"
  16 #include "AMDGPU.h"
  17 #include "AMDGPUSubtarget.h"
  18 #include "R600Defines.h"
  19 #include "R600InstrInfo.h"
  20 #include "R600MachineFunctionInfo.h"
  21 #include "R600RegisterInfo.h"
  22 #include "llvm/CodeGen/MachineFunctionPass.h"
  23 #include "llvm/CodeGen/MachineInstrBuilder.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/Support/raw_ostream.h"
  26
  27 using namespace llvm;
  28
  29 #define DEBUG_TYPE "r600cf"
  30
  31 namespace {
  32
  33 struct CFStack {
  34
  35   enum StackItem {
  36     ENTRY = 0,
  37     SUB_ENTRY = 1,
  38     FIRST_NON_WQM_PUSH = 2,
  39     FIRST_NON_WQM_PUSH_W_FULL_ENTRY = 3
  40   };
  41
  42   const AMDGPUSubtarget &ST;
  43   std::vector<StackItem> BranchStack;
  44   std::vector<StackItem> LoopStack;
  45   unsigned MaxStackSize;
  46   unsigned CurrentEntries;
  47   unsigned CurrentSubEntries;
  48
  49   CFStack(const AMDGPUSubtarget &st, unsigned ShaderType) : ST(st),
  50       // We need to reserve a stack entry for CALL_FS in vertex shaders.
  51       MaxStackSize(ShaderType == ShaderType::VERTEX ? 1 : 0),
  52       CurrentEntries(0), CurrentSubEntries(0) { }
  53
  54   unsigned getLoopDepth();
  55   bool branchStackContains(CFStack::StackItem);
  56   bool requiresWorkAroundForInst(unsigned Opcode);
  57   unsigned getSubEntrySize(CFStack::StackItem Item);
  58   void updateMaxStackSize();
  59   void pushBranch(unsigned Opcode, bool isWQM = false);
  60   void pushLoop();
  61   void popBranch();
  62   void popLoop();
  63 };
  64
  65 unsigned CFStack::getLoopDepth() {
  66   return LoopStack.size();
  67 }
  68
  69 bool CFStack::branchStackContains(CFStack::StackItem Item) {
  70   for (std::vector<CFStack::StackItem>::const_iterator I = BranchStack.begin(),
  71        E = BranchStack.end(); I != E; ++I) {
  72     if (*I == Item)
  73       return true;
  74   }
  75   return false;
  76 }
  77
  78 bool CFStack::requiresWorkAroundForInst(unsigned Opcode) {
  79   if (Opcode == AMDGPU::CF_ALU_PUSH_BEFORE && ST.hasCaymanISA() &&
  80       getLoopDepth() > 1)
  81     return true;
  82
  83   if (!ST.hasCFAluBug())
  84     return false;
  85
  86   switch(Opcode) {
  87   default: return false;
  88   case AMDGPU::CF_ALU_PUSH_BEFORE:
  89   case AMDGPU::CF_ALU_ELSE_AFTER:
  90   case AMDGPU::CF_ALU_BREAK:
  91   case AMDGPU::CF_ALU_CONTINUE:
  92     if (CurrentSubEntries == 0)
  93       return false;
  94     if (ST.getWavefrontSize() == 64) {
  95       // We are being conservative here.  We only require this work-around if
  96       // CurrentSubEntries > 3 &&
  97       // (CurrentSubEntries % 4 == 3 || CurrentSubEntries % 4 == 0)
  98       //
  99       // We have to be conservative, because we don't know for certain that
 100       // our stack allocation algorithm for Evergreen/NI is correct.  Applying this
 101       // work-around when CurrentSubEntries > 3 allows us to over-allocate stack
 102       // resources without any problems.
 103       return CurrentSubEntries > 3;
 104     } else {
 105       assert(ST.getWavefrontSize() == 32);
 106       // We are being conservative here.  We only require the work-around if
 107       // CurrentSubEntries > 7 &&
 108       // (CurrentSubEntries % 8 == 7 || CurrentSubEntries % 8 == 0)
 109       // See the comment on the wavefront size == 64 case for why we are
 110       // being conservative.
 111       return CurrentSubEntries > 7;
 112     }
 113   }
 114 }
 115
 116 unsigned CFStack::getSubEntrySize(CFStack::StackItem Item) {
 117   switch(Item) {
 118   default:
 119     return 0;
 120   case CFStack::FIRST_NON_WQM_PUSH:
 121   assert(!ST.hasCaymanISA());
 122   if (ST.getGeneration() <= AMDGPUSubtarget::R700) {
 123     // +1 For the push operation.
 124     // +2 Extra space required.
 125     return 3;
 126   } else {
 127     // Some documentation says that this is not necessary on Evergreen,
 128     // but experimentation has show that we need to allocate 1 extra
 129     // sub-entry for the first non-WQM push.
 130     // +1 For the push operation.
 131     // +1 Extra space required.
 132     return 2;
 133   }
 134   case CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY:
 135     assert(ST.getGeneration() >= AMDGPUSubtarget::EVERGREEN);
 136     // +1 For the push operation.
 137     // +1 Extra space required.
 138     return 2;
 139   case CFStack::SUB_ENTRY:
 140     return 1;
 141   }
 142 }
 143
 144 void CFStack::updateMaxStackSize() {
 145   unsigned CurrentStackSize = CurrentEntries +
 146                               (RoundUpToAlignment(CurrentSubEntries, 4) / 4);
 147   MaxStackSize = std::max(CurrentStackSize, MaxStackSize);
 148 }
 149
 150 void CFStack::pushBranch(unsigned Opcode, bool isWQM) {
 151   CFStack::StackItem Item = CFStack::ENTRY;
 152   switch(Opcode) {
 153   case AMDGPU::CF_PUSH_EG:
 154   case AMDGPU::CF_ALU_PUSH_BEFORE:
 155     if (!isWQM) {
 156       if (!ST.hasCaymanISA() && !branchStackContains(CFStack::FIRST_NON_WQM_PUSH))
 157         Item = CFStack::FIRST_NON_WQM_PUSH;  // May not be required on Evergreen/NI
 158                                              // See comment in
 159                                              // CFStack::getSubEntrySize()
 160       else if (CurrentEntries > 0 &&
 161                ST.getGeneration() > AMDGPUSubtarget::EVERGREEN &&
 162                !ST.hasCaymanISA() &&
 163                !branchStackContains(CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY))
 164         Item = CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY;
 165       else
 166         Item = CFStack::SUB_ENTRY;
 167     } else
 168       Item = CFStack::ENTRY;
 169     break;
 170   }
 171   BranchStack.push_back(Item);
 172   if (Item == CFStack::ENTRY)
 173     CurrentEntries++;
 174   else
 175     CurrentSubEntries += getSubEntrySize(Item);
 176   updateMaxStackSize();
 177 }
 178
 179 void CFStack::pushLoop() {
 180   LoopStack.push_back(CFStack::ENTRY);
 181   CurrentEntries++;
 182   updateMaxStackSize();
 183 }
 184
 185 void CFStack::popBranch() {
 186   CFStack::StackItem Top = BranchStack.back();
 187   if (Top == CFStack::ENTRY)
 188     CurrentEntries--;
 189   else
 190     CurrentSubEntries-= getSubEntrySize(Top);
 191   BranchStack.pop_back();
 192 }
 193
 194 void CFStack::popLoop() {
 195   CurrentEntries--;
 196   LoopStack.pop_back();
 197 }
 198
 199 class R600ControlFlowFinalizer : public MachineFunctionPass {
 200
 201 private:
 202   typedef std::pair<MachineInstr *, std::vector<MachineInstr *> > ClauseFile;
 203
 204   enum ControlFlowInstruction {
 205     CF_TC,
 206     CF_VC,
 207     CF_CALL_FS,
 208     CF_WHILE_LOOP,
 209     CF_END_LOOP,
 210     CF_LOOP_BREAK,
 211     CF_LOOP_CONTINUE,
 212     CF_JUMP,
 213     CF_ELSE,
 214     CF_POP,
 215     CF_END
 216   };
 217
 218   static char ID;
 219   const R600InstrInfo *TII;
 220   const R600RegisterInfo *TRI;
 221   unsigned MaxFetchInst;
 222   const AMDGPUSubtarget &ST;
 223
 224   bool IsTrivialInst(MachineInstr *MI) const {
 225     switch (MI->getOpcode()) {
 226     case AMDGPU::KILL:
 227     case AMDGPU::RETURN:
 228       return true;
 229     default:
 230       return false;
 231     }
 232   }
 233
 234   const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
 235     unsigned Opcode = 0;
 236     bool isEg = (ST.getGeneration() >= AMDGPUSubtarget::EVERGREEN);
 237     switch (CFI) {
 238     case CF_TC:
 239       Opcode = isEg ? AMDGPU::CF_TC_EG : AMDGPU::CF_TC_R600;
 240       break;
 241     case CF_VC:
 242       Opcode = isEg ? AMDGPU::CF_VC_EG : AMDGPU::CF_VC_R600;
 243       break;
 244     case CF_CALL_FS:
 245       Opcode = isEg ? AMDGPU::CF_CALL_FS_EG : AMDGPU::CF_CALL_FS_R600;
 246       break;
 247     case CF_WHILE_LOOP:
 248       Opcode = isEg ? AMDGPU::WHILE_LOOP_EG : AMDGPU::WHILE_LOOP_R600;
 249       break;
 250     case CF_END_LOOP:
 251       Opcode = isEg ? AMDGPU::END_LOOP_EG : AMDGPU::END_LOOP_R600;
 252       break;
 253     case CF_LOOP_BREAK:
 254       Opcode = isEg ? AMDGPU::LOOP_BREAK_EG : AMDGPU::LOOP_BREAK_R600;
 255       break;
 256     case CF_LOOP_CONTINUE:
 257       Opcode = isEg ? AMDGPU::CF_CONTINUE_EG : AMDGPU::CF_CONTINUE_R600;
 258       break;
 259     case CF_JUMP:
 260       Opcode = isEg ? AMDGPU::CF_JUMP_EG : AMDGPU::CF_JUMP_R600;
 261       break;
 262     case CF_ELSE:
 263       Opcode = isEg ? AMDGPU::CF_ELSE_EG : AMDGPU::CF_ELSE_R600;
 264       break;
 265     case CF_POP:
 266       Opcode = isEg ? AMDGPU::POP_EG : AMDGPU::POP_R600;
 267       break;
 268     case CF_END:
 269       if (ST.hasCaymanISA()) {
 270         Opcode = AMDGPU::CF_END_CM;
 271         break;
 272       }
 273       Opcode = isEg ? AMDGPU::CF_END_EG : AMDGPU::CF_END_R600;
 274       break;
 275     }
 276     assert (Opcode && "No opcode selected");
 277     return TII->get(Opcode);
 278   }
 279
 280   bool isCompatibleWithClause(const MachineInstr *MI,
 281       std::set<unsigned> &DstRegs) const {
 282     unsigned DstMI, SrcMI;
 283     for (MachineInstr::const_mop_iterator I = MI->operands_begin(),
 284         E = MI->operands_end(); I != E; ++I) {
 285       const MachineOperand &MO = *I;
 286       if (!MO.isReg())
 287         continue;
 288       if (MO.isDef()) {
 289         unsigned Reg = MO.getReg();
 290         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 291           DstMI = Reg;
 292         else
 293           DstMI = TRI->getMatchingSuperReg(Reg,
 294               TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
 295               &AMDGPU::R600_Reg128RegClass);
 296       }
 297       if (MO.isUse()) {
 298         unsigned Reg = MO.getReg();
 299         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 300           SrcMI = Reg;
 301         else
 302           SrcMI = TRI->getMatchingSuperReg(Reg,
 303               TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
 304               &AMDGPU::R600_Reg128RegClass);
 305       }
 306     }
 307     if ((DstRegs.find(SrcMI) == DstRegs.end())) {
 308       DstRegs.insert(DstMI);
 309       return true;
 310     } else
 311       return false;
 312   }
 313
 314   ClauseFile
 315   MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 316       const {
 317     MachineBasicBlock::iterator ClauseHead = I;
 318     std::vector<MachineInstr *> ClauseContent;
 319     unsigned AluInstCount = 0;
 320     bool IsTex = TII->usesTextureCache(ClauseHead);
 321     std::set<unsigned> DstRegs;
 322     for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
 323       if (IsTrivialInst(I))
 324         continue;
 325       if (AluInstCount >= MaxFetchInst)
 326         break;
 327       if ((IsTex && !TII->usesTextureCache(I)) ||
 328           (!IsTex && !TII->usesVertexCache(I)))
 329         break;
 330       if (!isCompatibleWithClause(I, DstRegs))
 331         break;
 332       AluInstCount ++;
 333       ClauseContent.push_back(I);
 334     }
 335     MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
 336         getHWInstrDesc(IsTex?CF_TC:CF_VC))
 337         .addImm(0) // ADDR
 338         .addImm(AluInstCount - 1); // COUNT
 339     return ClauseFile(MIb, ClauseContent);
 340   }
 341
 342   void getLiteral(MachineInstr *MI, std::vector<int64_t> &Lits) const {
 343     static const unsigned LiteralRegs[] = {
 344       AMDGPU::ALU_LITERAL_X,
 345       AMDGPU::ALU_LITERAL_Y,
 346       AMDGPU::ALU_LITERAL_Z,
 347       AMDGPU::ALU_LITERAL_W
 348     };
 349     const SmallVector<std::pair<MachineOperand *, int64_t>, 3 > Srcs =
 350         TII->getSrcs(MI);
 351     for (unsigned i = 0, e = Srcs.size(); i < e; ++i) {
 352       if (Srcs[i].first->getReg() != AMDGPU::ALU_LITERAL_X)
 353         continue;
 354       int64_t Imm = Srcs[i].second;
 355       std::vector<int64_t>::iterator It =
 356           std::find(Lits.begin(), Lits.end(), Imm);
 357       if (It != Lits.end()) {
 358         unsigned Index = It - Lits.begin();
 359         Srcs[i].first->setReg(LiteralRegs[Index]);
 360       } else {
 361         assert(Lits.size() < 4 && "Too many literals in Instruction Group");
 362         Srcs[i].first->setReg(LiteralRegs[Lits.size()]);
 363         Lits.push_back(Imm);
 364       }
 365     }
 366   }
 367
 368   MachineBasicBlock::iterator insertLiterals(
 369       MachineBasicBlock::iterator InsertPos,
 370       const std::vector<unsigned> &Literals) const {
 371     MachineBasicBlock *MBB = InsertPos->getParent();
 372     for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 373       unsigned LiteralPair0 = Literals[i];
 374       unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
 375       InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
 376           TII->get(AMDGPU::LITERALS))
 377           .addImm(LiteralPair0)
 378           .addImm(LiteralPair1);
 379     }
 380     return InsertPos;
 381   }
 382
 383   ClauseFile
 384   MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 385       const {
 386     MachineBasicBlock::iterator ClauseHead = I;
 387     std::vector<MachineInstr *> ClauseContent;
 388     I++;
 389     for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
 390       if (IsTrivialInst(I)) {
 391         ++I;
 392         continue;
 393       }
 394       if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
 395         break;
 396       std::vector<int64_t> Literals;
 397       if (I->isBundle()) {
 398         MachineInstr *DeleteMI = I;
 399         MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
 400         while (++BI != E && BI->isBundledWithPred()) {
 401           BI->unbundleFromPred();
 402           for (unsigned i = 0, e = BI->getNumOperands(); i != e; ++i) {
 403             MachineOperand &MO = BI->getOperand(i);
 404             if (MO.isReg() && MO.isInternalRead())
 405               MO.setIsInternalRead(false);
 406           }
 407           getLiteral(BI, Literals);
 408           ClauseContent.push_back(BI);
 409         }
 410         I = BI;
 411         DeleteMI->eraseFromParent();
 412       } else {
 413         getLiteral(I, Literals);
 414         ClauseContent.push_back(I);
 415         I++;
 416       }
 417       for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 418         unsigned literal0 = Literals[i];
 419         unsigned literal2 = (i + 1 < e)?Literals[i + 1]:0;
 420         MachineInstr *MILit = BuildMI(MBB, I, I->getDebugLoc(),
 421             TII->get(AMDGPU::LITERALS))
 422             .addImm(literal0)
 423             .addImm(literal2);
 424         ClauseContent.push_back(MILit);
 425       }
 426     }
 427     assert(ClauseContent.size() < 128 && "ALU clause is too big");
 428     ClauseHead->getOperand(7).setImm(ClauseContent.size() - 1);
 429     return ClauseFile(ClauseHead, ClauseContent);
 430   }
 431
 432   void
 433   EmitFetchClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 434       unsigned &CfCount) {
 435     CounterPropagateAddr(Clause.first, CfCount);
 436     MachineBasicBlock *BB = Clause.first->getParent();
 437     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::FETCH_CLAUSE))
 438         .addImm(CfCount);
 439     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 440       BB->splice(InsertPos, BB, Clause.second[i]);
 441     }
 442     CfCount += 2 * Clause.second.size();
 443   }
 444
 445   void
 446   EmitALUClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 447       unsigned &CfCount) {
 448     Clause.first->getOperand(0).setImm(0);
 449     CounterPropagateAddr(Clause.first, CfCount);
 450     MachineBasicBlock *BB = Clause.first->getParent();
 451     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::ALU_CLAUSE))
 452         .addImm(CfCount);
 453     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 454       BB->splice(InsertPos, BB, Clause.second[i]);
 455     }
 456     CfCount += Clause.second.size();
 457   }
 458
 459   void CounterPropagateAddr(MachineInstr *MI, unsigned Addr) const {
 460     MI->getOperand(0).setImm(Addr + MI->getOperand(0).getImm());
 461   }
 462   void CounterPropagateAddr(std::set<MachineInstr *> MIs, unsigned Addr)
 463       const {
 464     for (std::set<MachineInstr *>::iterator It = MIs.begin(), E = MIs.end();
 465         It != E; ++It) {
 466       MachineInstr *MI = *It;
 467       CounterPropagateAddr(MI, Addr);
 468     }
 469   }
 470
 471 public:
 472   R600ControlFlowFinalizer(TargetMachine &tm) : MachineFunctionPass(ID),
 473     TII (nullptr), TRI(nullptr),
 474     ST(tm.getSubtarget<AMDGPUSubtarget>()) {
 475       const AMDGPUSubtarget &ST = tm.getSubtarget<AMDGPUSubtarget>();
 476       MaxFetchInst = ST.getTexVTXClauseSize();
 477   }
 478
 479   bool runOnMachineFunction(MachineFunction &MF) override {
 480     TII=static_cast<const R600InstrInfo *>(MF.getTarget().getInstrInfo());
 481     TRI=static_cast<const R600RegisterInfo *>(MF.getTarget().getRegisterInfo());
 482     R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
 483
 484     CFStack CFStack(ST, MFI->getShaderType());
 485     for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
 486         ++MB) {
 487       MachineBasicBlock &MBB = *MB;
 488       unsigned CfCount = 0;
 489       std::vector<std::pair<unsigned, std::set<MachineInstr *> > > LoopStack;
 490       std::vector<MachineInstr * > IfThenElseStack;
 491       if (MFI->getShaderType() == ShaderType::VERTEX) {
 492         BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
 493             getHWInstrDesc(CF_CALL_FS));
 494         CfCount++;
 495       }
 496       std::vector<ClauseFile> FetchClauses, AluClauses;
 497       std::vector<MachineInstr *> LastAlu(1);
 498       std::vector<MachineInstr *> ToPopAfter;
 499
 500       for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
 501           I != E;) {
 502         if (TII->usesTextureCache(I) || TII->usesVertexCache(I)) {
 503           DEBUG(dbgs() << CfCount << ":"; I->dump(););
 504           FetchClauses.push_back(MakeFetchClause(MBB, I));
 505           CfCount++;
 506           LastAlu.back() = nullptr;
 507           continue;
 508         }
 509
 510         MachineBasicBlock::iterator MI = I;
 511         if (MI->getOpcode() != AMDGPU::ENDIF)
 512           LastAlu.back() = nullptr;
 513         if (MI->getOpcode() == AMDGPU::CF_ALU)
 514           LastAlu.back() = MI;
 515         I++;
 516         bool RequiresWorkAround =
 517             CFStack.requiresWorkAroundForInst(MI->getOpcode());
 518         switch (MI->getOpcode()) {
 519         case AMDGPU::CF_ALU_PUSH_BEFORE:
 520           if (RequiresWorkAround) {
 521             DEBUG(dbgs() << "Applying bug work-around for ALU_PUSH_BEFORE\n");
 522             BuildMI(MBB, MI, MBB.findDebugLoc(MI), TII->get(AMDGPU::CF_PUSH_EG))
 523                 .addImm(CfCount + 1)
 524                 .addImm(1);
 525             MI->setDesc(TII->get(AMDGPU::CF_ALU));
 526             CfCount++;
 527             CFStack.pushBranch(AMDGPU::CF_PUSH_EG);
 528           } else
 529             CFStack.pushBranch(AMDGPU::CF_ALU_PUSH_BEFORE);
 530
 531         case AMDGPU::CF_ALU:
 532           I = MI;
 533           AluClauses.push_back(MakeALUClause(MBB, I));
 534           DEBUG(dbgs() << CfCount << ":"; MI->dump(););
 535           CfCount++;
 536           break;
 537         case AMDGPU::WHILELOOP: {
 538           CFStack.pushLoop();
 539           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 540               getHWInstrDesc(CF_WHILE_LOOP))
 541               .addImm(1);
 542           std::pair<unsigned, std::set<MachineInstr *> > Pair(CfCount,
 543               std::set<MachineInstr *>());
 544           Pair.second.insert(MIb);
 545           LoopStack.push_back(Pair);
 546           MI->eraseFromParent();
 547           CfCount++;
 548           break;
 549         }
 550         case AMDGPU::ENDLOOP: {
 551           CFStack.popLoop();
 552           std::pair<unsigned, std::set<MachineInstr *> > Pair =
 553               LoopStack.back();
 554           LoopStack.pop_back();
 555           CounterPropagateAddr(Pair.second, CfCount);
 556           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
 557               .addImm(Pair.first + 1);
 558           MI->eraseFromParent();
 559           CfCount++;
 560           break;
 561         }
 562         case AMDGPU::IF_PREDICATE_SET: {
 563           LastAlu.push_back(nullptr);
 564           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 565               getHWInstrDesc(CF_JUMP))
 566               .addImm(0)
 567               .addImm(0);
 568           IfThenElseStack.push_back(MIb);
 569           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 570           MI->eraseFromParent();
 571           CfCount++;
 572           break;
 573         }
 574         case AMDGPU::ELSE: {
 575           MachineInstr * JumpInst = IfThenElseStack.back();
 576           IfThenElseStack.pop_back();
 577           CounterPropagateAddr(JumpInst, CfCount);
 578           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 579               getHWInstrDesc(CF_ELSE))
 580               .addImm(0)
 581               .addImm(0);
 582           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 583           IfThenElseStack.push_back(MIb);
 584           MI->eraseFromParent();
 585           CfCount++;
 586           break;
 587         }
 588         case AMDGPU::ENDIF: {
 589           CFStack.popBranch();
 590           if (LastAlu.back()) {
 591             ToPopAfter.push_back(LastAlu.back());
 592           } else {
 593             MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 594                 getHWInstrDesc(CF_POP))
 595                 .addImm(CfCount + 1)
 596                 .addImm(1);
 597             (void)MIb;
 598             DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 599             CfCount++;
 600           }
 601
 602           MachineInstr *IfOrElseInst = IfThenElseStack.back();
 603           IfThenElseStack.pop_back();
 604           CounterPropagateAddr(IfOrElseInst, CfCount);
 605           IfOrElseInst->getOperand(1).setImm(1);
 606           LastAlu.pop_back();
 607           MI->eraseFromParent();
 608           break;
 609         }
 610         case AMDGPU::BREAK: {
 611           CfCount ++;
 612           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 613               getHWInstrDesc(CF_LOOP_BREAK))
 614               .addImm(0);
 615           LoopStack.back().second.insert(MIb);
 616           MI->eraseFromParent();
 617           break;
 618         }
 619         case AMDGPU::CONTINUE: {
 620           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 621               getHWInstrDesc(CF_LOOP_CONTINUE))
 622               .addImm(0);
 623           LoopStack.back().second.insert(MIb);
 624           MI->eraseFromParent();
 625           CfCount++;
 626           break;
 627         }
 628         case AMDGPU::RETURN: {
 629           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END));
 630           CfCount++;
 631           MI->eraseFromParent();
 632           if (CfCount % 2) {
 633             BuildMI(MBB, I, MBB.findDebugLoc(MI), TII->get(AMDGPU::PAD));
 634             CfCount++;
 635           }
 636           for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
 637             EmitFetchClause(I, FetchClauses[i], CfCount);
 638           for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
 639             EmitALUClause(I, AluClauses[i], CfCount);
 640         }
 641         default:
 642           if (TII->isExport(MI->getOpcode())) {
 643             DEBUG(dbgs() << CfCount << ":"; MI->dump(););
 644             CfCount++;
 645           }
 646           break;
 647         }
 648       }
 649       for (unsigned i = 0, e = ToPopAfter.size(); i < e; ++i) {
 650         MachineInstr *Alu = ToPopAfter[i];
 651         BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu),
 652             TII->get(AMDGPU::CF_ALU_POP_AFTER))
 653             .addImm(Alu->getOperand(0).getImm())
 654             .addImm(Alu->getOperand(1).getImm())
 655             .addImm(Alu->getOperand(2).getImm())
 656             .addImm(Alu->getOperand(3).getImm())
 657             .addImm(Alu->getOperand(4).getImm())
 658             .addImm(Alu->getOperand(5).getImm())
 659             .addImm(Alu->getOperand(6).getImm())
 660             .addImm(Alu->getOperand(7).getImm())
 661             .addImm(Alu->getOperand(8).getImm());
 662         Alu->eraseFromParent();
 663       }
 664       MFI->StackSize = CFStack.MaxStackSize;
 665     }
 666
 667     return false;
 668   }
 669
 670   const char *getPassName() const override {
 671     return "R600 Control Flow Finalizer Pass";
 672   }
 673 };
 674
 675 char R600ControlFlowFinalizer::ID = 0;
 676
 677 } // end anonymous namespace
 678
 679
 680 llvm::FunctionPass *llvm::createR600ControlFlowFinalizer(TargetMachine &TM) {
 681   return new R600ControlFlowFinalizer(TM);
 682 }