lib/Target/R600/R600ControlFlowFinalizer.cpp

   1 //===-- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst----------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 /// \file
  11 /// This pass compute turns all control flow pseudo instructions into native one
  12 /// computing their address on the fly ; it also sets STACK_SIZE info.
  13 //===----------------------------------------------------------------------===//
  14
  15 #define DEBUG_TYPE "r600cf"
  16 #include "llvm/Support/Debug.h"
  17 #include "AMDGPU.h"
  18 #include "R600Defines.h"
  19 #include "R600InstrInfo.h"
  20 #include "R600MachineFunctionInfo.h"
  21 #include "R600RegisterInfo.h"
  22 #include "llvm/CodeGen/MachineFunctionPass.h"
  23 #include "llvm/CodeGen/MachineInstrBuilder.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/Support/raw_ostream.h"
  26
  27 using namespace llvm;
  28
  29 namespace {
  30
  31 class R600ControlFlowFinalizer : public MachineFunctionPass {
  32
  33 private:
  34   typedef std::pair<MachineInstr *, std::vector<MachineInstr *> > ClauseFile;
  35
  36   enum ControlFlowInstruction {
  37     CF_TC,
  38     CF_VC,
  39     CF_CALL_FS,
  40     CF_WHILE_LOOP,
  41     CF_END_LOOP,
  42     CF_LOOP_BREAK,
  43     CF_LOOP_CONTINUE,
  44     CF_JUMP,
  45     CF_ELSE,
  46     CF_POP,
  47     CF_END
  48   };
  49
  50   static char ID;
  51   const R600InstrInfo *TII;
  52   const R600RegisterInfo &TRI;
  53   unsigned MaxFetchInst;
  54   const AMDGPUSubtarget &ST;
  55
  56   bool IsTrivialInst(MachineInstr *MI) const {
  57     switch (MI->getOpcode()) {
  58     case AMDGPU::KILL:
  59     case AMDGPU::RETURN:
  60       return true;
  61     default:
  62       return false;
  63     }
  64   }
  65
  66   const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
  67     unsigned Opcode = 0;
  68     bool isEg = (ST.device()->getGeneration() >= AMDGPUDeviceInfo::HD5XXX);
  69     switch (CFI) {
  70     case CF_TC:
  71       Opcode = isEg ? AMDGPU::CF_TC_EG : AMDGPU::CF_TC_R600;
  72       break;
  73     case CF_VC:
  74       Opcode = isEg ? AMDGPU::CF_VC_EG : AMDGPU::CF_VC_R600;
  75       break;
  76     case CF_CALL_FS:
  77       Opcode = isEg ? AMDGPU::CF_CALL_FS_EG : AMDGPU::CF_CALL_FS_R600;
  78       break;
  79     case CF_WHILE_LOOP:
  80       Opcode = isEg ? AMDGPU::WHILE_LOOP_EG : AMDGPU::WHILE_LOOP_R600;
  81       break;
  82     case CF_END_LOOP:
  83       Opcode = isEg ? AMDGPU::END_LOOP_EG : AMDGPU::END_LOOP_R600;
  84       break;
  85     case CF_LOOP_BREAK:
  86       Opcode = isEg ? AMDGPU::LOOP_BREAK_EG : AMDGPU::LOOP_BREAK_R600;
  87       break;
  88     case CF_LOOP_CONTINUE:
  89       Opcode = isEg ? AMDGPU::CF_CONTINUE_EG : AMDGPU::CF_CONTINUE_R600;
  90       break;
  91     case CF_JUMP:
  92       Opcode = isEg ? AMDGPU::CF_JUMP_EG : AMDGPU::CF_JUMP_R600;
  93       break;
  94     case CF_ELSE:
  95       Opcode = isEg ? AMDGPU::CF_ELSE_EG : AMDGPU::CF_ELSE_R600;
  96       break;
  97     case CF_POP:
  98       Opcode = isEg ? AMDGPU::POP_EG : AMDGPU::POP_R600;
  99       break;
 100     case CF_END:
 101       if (ST.device()->getDeviceFlag() == OCL_DEVICE_CAYMAN) {
 102         Opcode = AMDGPU::CF_END_CM;
 103         break;
 104       }
 105       Opcode = isEg ? AMDGPU::CF_END_EG : AMDGPU::CF_END_R600;
 106       break;
 107     }
 108     assert (Opcode && "No opcode selected");
 109     return TII->get(Opcode);
 110   }
 111
 112   bool isCompatibleWithClause(const MachineInstr *MI,
 113   std::set<unsigned> &DstRegs, std::set<unsigned> &SrcRegs) const {
 114     unsigned DstMI, SrcMI;
 115     for (MachineInstr::const_mop_iterator I = MI->operands_begin(),
 116         E = MI->operands_end(); I != E; ++I) {
 117       const MachineOperand &MO = *I;
 118       if (!MO.isReg())
 119         continue;
 120       if (MO.isDef()) {
 121         unsigned Reg = MO.getReg();
 122         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 123           DstMI = Reg;
 124         else
 125           DstMI = TRI.getMatchingSuperReg(Reg,
 126               TRI.getSubRegFromChannel(TRI.getHWRegChan(Reg)),
 127               &AMDGPU::R600_Reg128RegClass);
 128       }
 129       if (MO.isUse()) {
 130         unsigned Reg = MO.getReg();
 131         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 132           SrcMI = Reg;
 133         else
 134           SrcMI = TRI.getMatchingSuperReg(Reg,
 135               TRI.getSubRegFromChannel(TRI.getHWRegChan(Reg)),
 136               &AMDGPU::R600_Reg128RegClass);
 137       }
 138     }
 139     if ((DstRegs.find(SrcMI) == DstRegs.end()) &&
 140         (SrcRegs.find(DstMI) == SrcRegs.end())) {
 141       SrcRegs.insert(SrcMI);
 142       DstRegs.insert(DstMI);
 143       return true;
 144     } else
 145       return false;
 146   }
 147
 148   ClauseFile
 149   MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 150       const {
 151     MachineBasicBlock::iterator ClauseHead = I;
 152     std::vector<MachineInstr *> ClauseContent;
 153     unsigned AluInstCount = 0;
 154     bool IsTex = TII->usesTextureCache(ClauseHead);
 155     std::set<unsigned> DstRegs, SrcRegs;
 156     for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
 157       if (IsTrivialInst(I))
 158         continue;
 159       if (AluInstCount >= MaxFetchInst)
 160         break;
 161       if ((IsTex && !TII->usesTextureCache(I)) ||
 162           (!IsTex && !TII->usesVertexCache(I)))
 163         break;
 164       if (!isCompatibleWithClause(I, DstRegs, SrcRegs))
 165         break;
 166       AluInstCount ++;
 167       ClauseContent.push_back(I);
 168     }
 169     MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
 170         getHWInstrDesc(IsTex?CF_TC:CF_VC))
 171         .addImm(0) // ADDR
 172         .addImm(AluInstCount - 1); // COUNT
 173     return ClauseFile(MIb, ClauseContent);
 174   }
 175
 176   void getLiteral(MachineInstr *MI, std::vector<int64_t> &Lits) const {
 177     unsigned LiteralRegs[] = {
 178       AMDGPU::ALU_LITERAL_X,
 179       AMDGPU::ALU_LITERAL_Y,
 180       AMDGPU::ALU_LITERAL_Z,
 181       AMDGPU::ALU_LITERAL_W
 182     };
 183     const SmallVector<std::pair<MachineOperand *, int64_t>, 3 > Srcs =
 184         TII->getSrcs(MI);
 185     for (unsigned i = 0, e = Srcs.size(); i < e; ++i) {
 186       if (Srcs[i].first->getReg() != AMDGPU::ALU_LITERAL_X)
 187         continue;
 188       int64_t Imm = Srcs[i].second;
 189       std::vector<int64_t>::iterator It =
 190           std::find(Lits.begin(), Lits.end(), Imm);
 191       if (It != Lits.end()) {
 192         unsigned Index = It - Lits.begin();
 193         Srcs[i].first->setReg(LiteralRegs[Index]);
 194       } else {
 195         assert(Lits.size() < 4 && "Too many literals in Instruction Group");
 196         Srcs[i].first->setReg(LiteralRegs[Lits.size()]);
 197         Lits.push_back(Imm);
 198       }
 199     }
 200   }
 201
 202   MachineBasicBlock::iterator insertLiterals(
 203       MachineBasicBlock::iterator InsertPos,
 204       const std::vector<unsigned> &Literals) const {
 205     MachineBasicBlock *MBB = InsertPos->getParent();
 206     for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 207       unsigned LiteralPair0 = Literals[i];
 208       unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
 209       InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
 210           TII->get(AMDGPU::LITERALS))
 211           .addImm(LiteralPair0)
 212           .addImm(LiteralPair1);
 213     }
 214     return InsertPos;
 215   }
 216
 217   ClauseFile
 218   MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 219       const {
 220     MachineBasicBlock::iterator ClauseHead = I;
 221     std::vector<MachineInstr *> ClauseContent;
 222     I++;
 223     for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
 224       if (IsTrivialInst(I)) {
 225         ++I;
 226         continue;
 227       }
 228       if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
 229         break;
 230       std::vector<int64_t> Literals;
 231       if (I->isBundle()) {
 232         MachineInstr *DeleteMI = I;
 233         MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
 234         while (++BI != E && BI->isBundledWithPred()) {
 235           BI->unbundleFromPred();
 236           for (unsigned i = 0, e = BI->getNumOperands(); i != e; ++i) {
 237             MachineOperand &MO = BI->getOperand(i);
 238             if (MO.isReg() && MO.isInternalRead())
 239               MO.setIsInternalRead(false);
 240           }
 241           getLiteral(BI, Literals);
 242           ClauseContent.push_back(BI);
 243         }
 244         I = BI;
 245         DeleteMI->eraseFromParent();
 246       } else {
 247         getLiteral(I, Literals);
 248         ClauseContent.push_back(I);
 249         I++;
 250       }
 251       for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 252         unsigned literal0 = Literals[i];
 253         unsigned literal2 = (i + 1 < e)?Literals[i + 1]:0;
 254         MachineInstr *MILit = BuildMI(MBB, I, I->getDebugLoc(),
 255             TII->get(AMDGPU::LITERALS))
 256             .addImm(literal0)
 257             .addImm(literal2);
 258         ClauseContent.push_back(MILit);
 259       }
 260     }
 261     ClauseHead->getOperand(7).setImm(ClauseContent.size() - 1);
 262     return ClauseFile(ClauseHead, ClauseContent);
 263   }
 264
 265   void
 266   EmitFetchClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 267       unsigned &CfCount) {
 268     CounterPropagateAddr(Clause.first, CfCount);
 269     MachineBasicBlock *BB = Clause.first->getParent();
 270     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::FETCH_CLAUSE))
 271         .addImm(CfCount);
 272     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 273       BB->splice(InsertPos, BB, Clause.second[i]);
 274     }
 275     CfCount += 2 * Clause.second.size();
 276   }
 277
 278   void
 279   EmitALUClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 280       unsigned &CfCount) {
 281     CounterPropagateAddr(Clause.first, CfCount);
 282     MachineBasicBlock *BB = Clause.first->getParent();
 283     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::ALU_CLAUSE))
 284         .addImm(CfCount);
 285     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 286       BB->splice(InsertPos, BB, Clause.second[i]);
 287     }
 288     CfCount += Clause.second.size();
 289   }
 290
 291   void CounterPropagateAddr(MachineInstr *MI, unsigned Addr) const {
 292     MI->getOperand(0).setImm(Addr + MI->getOperand(0).getImm());
 293   }
 294   void CounterPropagateAddr(std::set<MachineInstr *> MIs, unsigned Addr)
 295       const {
 296     for (std::set<MachineInstr *>::iterator It = MIs.begin(), E = MIs.end();
 297         It != E; ++It) {
 298       MachineInstr *MI = *It;
 299       CounterPropagateAddr(MI, Addr);
 300     }
 301   }
 302
 303   unsigned getHWStackSize(unsigned StackSubEntry, bool hasPush) const {
 304     switch (ST.device()->getGeneration()) {
 305     case AMDGPUDeviceInfo::HD4XXX:
 306       if (hasPush)
 307         StackSubEntry += 2;
 308       break;
 309     case AMDGPUDeviceInfo::HD5XXX:
 310       if (hasPush)
 311         StackSubEntry ++;
 312     case AMDGPUDeviceInfo::HD6XXX:
 313       StackSubEntry += 2;
 314       break;
 315     }
 316     return (StackSubEntry + 3)/4; // Need ceil value of StackSubEntry/4
 317   }
 318
 319 public:
 320   R600ControlFlowFinalizer(TargetMachine &tm) : MachineFunctionPass(ID),
 321     TII (static_cast<const R600InstrInfo *>(tm.getInstrInfo())),
 322     TRI(TII->getRegisterInfo()),
 323     ST(tm.getSubtarget<AMDGPUSubtarget>()) {
 324       const AMDGPUSubtarget &ST = tm.getSubtarget<AMDGPUSubtarget>();
 325       MaxFetchInst = ST.getTexVTXClauseSize();
 326   }
 327
 328   virtual bool runOnMachineFunction(MachineFunction &MF) {
 329     unsigned MaxStack = 0;
 330     unsigned CurrentStack = 0;
 331     bool HasPush = false;
 332     for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
 333         ++MB) {
 334       MachineBasicBlock &MBB = *MB;
 335       unsigned CfCount = 0;
 336       std::vector<std::pair<unsigned, std::set<MachineInstr *> > > LoopStack;
 337       std::vector<MachineInstr * > IfThenElseStack;
 338       R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
 339       if (MFI->ShaderType == 1) {
 340         BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
 341             getHWInstrDesc(CF_CALL_FS));
 342         CfCount++;
 343       }
 344       std::vector<ClauseFile> FetchClauses, AluClauses;
 345       for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
 346           I != E;) {
 347         if (TII->usesTextureCache(I) || TII->usesVertexCache(I)) {
 348           DEBUG(dbgs() << CfCount << ":"; I->dump(););
 349           FetchClauses.push_back(MakeFetchClause(MBB, I));
 350           CfCount++;
 351           continue;
 352         }
 353
 354         MachineBasicBlock::iterator MI = I;
 355         I++;
 356         switch (MI->getOpcode()) {
 357         case AMDGPU::CF_ALU_PUSH_BEFORE:
 358           CurrentStack++;
 359           MaxStack = std::max(MaxStack, CurrentStack);
 360           HasPush = true;
 361         case AMDGPU::CF_ALU:
 362           I = MI;
 363           AluClauses.push_back(MakeALUClause(MBB, I));
 364         case AMDGPU::EG_ExportBuf:
 365         case AMDGPU::EG_ExportSwz:
 366         case AMDGPU::R600_ExportBuf:
 367         case AMDGPU::R600_ExportSwz:
 368         case AMDGPU::RAT_WRITE_CACHELESS_32_eg:
 369         case AMDGPU::RAT_WRITE_CACHELESS_128_eg:
 370           DEBUG(dbgs() << CfCount << ":"; MI->dump(););
 371           CfCount++;
 372           break;
 373         case AMDGPU::WHILELOOP: {
 374           CurrentStack+=4;
 375           MaxStack = std::max(MaxStack, CurrentStack);
 376           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 377               getHWInstrDesc(CF_WHILE_LOOP))
 378               .addImm(1);
 379           std::pair<unsigned, std::set<MachineInstr *> > Pair(CfCount,
 380               std::set<MachineInstr *>());
 381           Pair.second.insert(MIb);
 382           LoopStack.push_back(Pair);
 383           MI->eraseFromParent();
 384           CfCount++;
 385           break;
 386         }
 387         case AMDGPU::ENDLOOP: {
 388           CurrentStack-=4;
 389           std::pair<unsigned, std::set<MachineInstr *> > Pair =
 390               LoopStack.back();
 391           LoopStack.pop_back();
 392           CounterPropagateAddr(Pair.second, CfCount);
 393           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
 394               .addImm(Pair.first + 1);
 395           MI->eraseFromParent();
 396           CfCount++;
 397           break;
 398         }
 399         case AMDGPU::IF_PREDICATE_SET: {
 400           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 401               getHWInstrDesc(CF_JUMP))
 402               .addImm(0)
 403               .addImm(0);
 404           IfThenElseStack.push_back(MIb);
 405           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 406           MI->eraseFromParent();
 407           CfCount++;
 408           break;
 409         }
 410         case AMDGPU::ELSE: {
 411           MachineInstr * JumpInst = IfThenElseStack.back();
 412           IfThenElseStack.pop_back();
 413           CounterPropagateAddr(JumpInst, CfCount);
 414           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 415               getHWInstrDesc(CF_ELSE))
 416               .addImm(0)
 417               .addImm(1);
 418           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 419           IfThenElseStack.push_back(MIb);
 420           MI->eraseFromParent();
 421           CfCount++;
 422           break;
 423         }
 424         case AMDGPU::ENDIF: {
 425           CurrentStack--;
 426           MachineInstr *IfOrElseInst = IfThenElseStack.back();
 427           IfThenElseStack.pop_back();
 428           CounterPropagateAddr(IfOrElseInst, CfCount + 1);
 429           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 430               getHWInstrDesc(CF_POP))
 431               .addImm(CfCount + 1)
 432               .addImm(1);
 433           (void)MIb;
 434           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 435           MI->eraseFromParent();
 436           CfCount++;
 437           break;
 438         }
 439         case AMDGPU::PREDICATED_BREAK: {
 440           CurrentStack--;
 441           CfCount += 3;
 442           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_JUMP))
 443               .addImm(CfCount)
 444               .addImm(1);
 445           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 446               getHWInstrDesc(CF_LOOP_BREAK))
 447               .addImm(0);
 448           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_POP))
 449               .addImm(CfCount)
 450               .addImm(1);
 451           LoopStack.back().second.insert(MIb);
 452           MI->eraseFromParent();
 453           break;
 454         }
 455         case AMDGPU::CONTINUE: {
 456           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 457               getHWInstrDesc(CF_LOOP_CONTINUE))
 458               .addImm(0);
 459           LoopStack.back().second.insert(MIb);
 460           MI->eraseFromParent();
 461           CfCount++;
 462           break;
 463         }
 464         case AMDGPU::RETURN: {
 465           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END));
 466           CfCount++;
 467           MI->eraseFromParent();
 468           if (CfCount % 2) {
 469             BuildMI(MBB, I, MBB.findDebugLoc(MI), TII->get(AMDGPU::PAD));
 470             CfCount++;
 471           }
 472           for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
 473             EmitFetchClause(I, FetchClauses[i], CfCount);
 474           for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
 475             EmitALUClause(I, AluClauses[i], CfCount);
 476         }
 477         default:
 478           break;
 479         }
 480       }
 481       MFI->StackSize = getHWStackSize(MaxStack, HasPush);
 482     }
 483
 484     return false;
 485   }
 486
 487   const char *getPassName() const {
 488     return "R600 Control Flow Finalizer Pass";
 489   }
 490 };
 491
 492 char R600ControlFlowFinalizer::ID = 0;
 493
 494 } // end anonymous namespace
 495
 496
 497 llvm::FunctionPass *llvm::createR600ControlFlowFinalizer(TargetMachine &TM) {
 498   return new R600ControlFlowFinalizer(TM);
 499 }