lib/Target/R600/R600ControlFlowFinalizer.cpp

   1 //===-- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst----------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 /// \file
  11 /// This pass compute turns all control flow pseudo instructions into native one
  12 /// computing their address on the fly ; it also sets STACK_SIZE info.
  13 //===----------------------------------------------------------------------===//
  14
  15 #define DEBUG_TYPE "r600cf"
  16 #include "llvm/Support/Debug.h"
  17 #include "AMDGPU.h"
  18 #include "R600Defines.h"
  19 #include "R600InstrInfo.h"
  20 #include "R600MachineFunctionInfo.h"
  21 #include "R600RegisterInfo.h"
  22 #include "llvm/CodeGen/MachineFunctionPass.h"
  23 #include "llvm/CodeGen/MachineInstrBuilder.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/Support/raw_ostream.h"
  26
  27 using namespace llvm;
  28
  29 namespace {
  30
  31 class R600ControlFlowFinalizer : public MachineFunctionPass {
  32
  33 private:
  34   typedef std::pair<MachineInstr *, std::vector<MachineInstr *> > ClauseFile;
  35
  36   enum ControlFlowInstruction {
  37     CF_TC,
  38     CF_VC,
  39     CF_CALL_FS,
  40     CF_WHILE_LOOP,
  41     CF_END_LOOP,
  42     CF_LOOP_BREAK,
  43     CF_LOOP_CONTINUE,
  44     CF_JUMP,
  45     CF_ELSE,
  46     CF_POP,
  47     CF_END
  48   };
  49
  50   static char ID;
  51   const R600InstrInfo *TII;
  52   const R600RegisterInfo *TRI;
  53   unsigned MaxFetchInst;
  54   const AMDGPUSubtarget &ST;
  55
  56   bool IsTrivialInst(MachineInstr *MI) const {
  57     switch (MI->getOpcode()) {
  58     case AMDGPU::KILL:
  59     case AMDGPU::RETURN:
  60       return true;
  61     default:
  62       return false;
  63     }
  64   }
  65
  66   const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
  67     unsigned Opcode = 0;
  68     bool isEg = (ST.getGeneration() >= AMDGPUSubtarget::EVERGREEN);
  69     switch (CFI) {
  70     case CF_TC:
  71       Opcode = isEg ? AMDGPU::CF_TC_EG : AMDGPU::CF_TC_R600;
  72       break;
  73     case CF_VC:
  74       Opcode = isEg ? AMDGPU::CF_VC_EG : AMDGPU::CF_VC_R600;
  75       break;
  76     case CF_CALL_FS:
  77       Opcode = isEg ? AMDGPU::CF_CALL_FS_EG : AMDGPU::CF_CALL_FS_R600;
  78       break;
  79     case CF_WHILE_LOOP:
  80       Opcode = isEg ? AMDGPU::WHILE_LOOP_EG : AMDGPU::WHILE_LOOP_R600;
  81       break;
  82     case CF_END_LOOP:
  83       Opcode = isEg ? AMDGPU::END_LOOP_EG : AMDGPU::END_LOOP_R600;
  84       break;
  85     case CF_LOOP_BREAK:
  86       Opcode = isEg ? AMDGPU::LOOP_BREAK_EG : AMDGPU::LOOP_BREAK_R600;
  87       break;
  88     case CF_LOOP_CONTINUE:
  89       Opcode = isEg ? AMDGPU::CF_CONTINUE_EG : AMDGPU::CF_CONTINUE_R600;
  90       break;
  91     case CF_JUMP:
  92       Opcode = isEg ? AMDGPU::CF_JUMP_EG : AMDGPU::CF_JUMP_R600;
  93       break;
  94     case CF_ELSE:
  95       Opcode = isEg ? AMDGPU::CF_ELSE_EG : AMDGPU::CF_ELSE_R600;
  96       break;
  97     case CF_POP:
  98       Opcode = isEg ? AMDGPU::POP_EG : AMDGPU::POP_R600;
  99       break;
 100     case CF_END:
 101       if (ST.hasCaymanISA()) {
 102         Opcode = AMDGPU::CF_END_CM;
 103         break;
 104       }
 105       Opcode = isEg ? AMDGPU::CF_END_EG : AMDGPU::CF_END_R600;
 106       break;
 107     }
 108     assert (Opcode && "No opcode selected");
 109     return TII->get(Opcode);
 110   }
 111
 112   bool isCompatibleWithClause(const MachineInstr *MI,
 113   std::set<unsigned> &DstRegs, std::set<unsigned> &SrcRegs) const {
 114     unsigned DstMI, SrcMI;
 115     for (MachineInstr::const_mop_iterator I = MI->operands_begin(),
 116         E = MI->operands_end(); I != E; ++I) {
 117       const MachineOperand &MO = *I;
 118       if (!MO.isReg())
 119         continue;
 120       if (MO.isDef()) {
 121         unsigned Reg = MO.getReg();
 122         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 123           DstMI = Reg;
 124         else
 125           DstMI = TRI->getMatchingSuperReg(Reg,
 126               TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
 127               &AMDGPU::R600_Reg128RegClass);
 128       }
 129       if (MO.isUse()) {
 130         unsigned Reg = MO.getReg();
 131         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 132           SrcMI = Reg;
 133         else
 134           SrcMI = TRI->getMatchingSuperReg(Reg,
 135               TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
 136               &AMDGPU::R600_Reg128RegClass);
 137       }
 138     }
 139     if ((DstRegs.find(SrcMI) == DstRegs.end()) &&
 140         (SrcRegs.find(DstMI) == SrcRegs.end())) {
 141       SrcRegs.insert(SrcMI);
 142       DstRegs.insert(DstMI);
 143       return true;
 144     } else
 145       return false;
 146   }
 147
 148   ClauseFile
 149   MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 150       const {
 151     MachineBasicBlock::iterator ClauseHead = I;
 152     std::vector<MachineInstr *> ClauseContent;
 153     unsigned AluInstCount = 0;
 154     bool IsTex = TII->usesTextureCache(ClauseHead);
 155     std::set<unsigned> DstRegs, SrcRegs;
 156     for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
 157       if (IsTrivialInst(I))
 158         continue;
 159       if (AluInstCount >= MaxFetchInst)
 160         break;
 161       if ((IsTex && !TII->usesTextureCache(I)) ||
 162           (!IsTex && !TII->usesVertexCache(I)))
 163         break;
 164       if (!isCompatibleWithClause(I, DstRegs, SrcRegs))
 165         break;
 166       AluInstCount ++;
 167       ClauseContent.push_back(I);
 168     }
 169     MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
 170         getHWInstrDesc(IsTex?CF_TC:CF_VC))
 171         .addImm(0) // ADDR
 172         .addImm(AluInstCount - 1); // COUNT
 173     return ClauseFile(MIb, ClauseContent);
 174   }
 175
 176   void getLiteral(MachineInstr *MI, std::vector<int64_t> &Lits) const {
 177     unsigned LiteralRegs[] = {
 178       AMDGPU::ALU_LITERAL_X,
 179       AMDGPU::ALU_LITERAL_Y,
 180       AMDGPU::ALU_LITERAL_Z,
 181       AMDGPU::ALU_LITERAL_W
 182     };
 183     const SmallVector<std::pair<MachineOperand *, int64_t>, 3 > Srcs =
 184         TII->getSrcs(MI);
 185     for (unsigned i = 0, e = Srcs.size(); i < e; ++i) {
 186       if (Srcs[i].first->getReg() != AMDGPU::ALU_LITERAL_X)
 187         continue;
 188       int64_t Imm = Srcs[i].second;
 189       std::vector<int64_t>::iterator It =
 190           std::find(Lits.begin(), Lits.end(), Imm);
 191       if (It != Lits.end()) {
 192         unsigned Index = It - Lits.begin();
 193         Srcs[i].first->setReg(LiteralRegs[Index]);
 194       } else {
 195         assert(Lits.size() < 4 && "Too many literals in Instruction Group");
 196         Srcs[i].first->setReg(LiteralRegs[Lits.size()]);
 197         Lits.push_back(Imm);
 198       }
 199     }
 200   }
 201
 202   MachineBasicBlock::iterator insertLiterals(
 203       MachineBasicBlock::iterator InsertPos,
 204       const std::vector<unsigned> &Literals) const {
 205     MachineBasicBlock *MBB = InsertPos->getParent();
 206     for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 207       unsigned LiteralPair0 = Literals[i];
 208       unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
 209       InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
 210           TII->get(AMDGPU::LITERALS))
 211           .addImm(LiteralPair0)
 212           .addImm(LiteralPair1);
 213     }
 214     return InsertPos;
 215   }
 216
 217   ClauseFile
 218   MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 219       const {
 220     MachineBasicBlock::iterator ClauseHead = I;
 221     std::vector<MachineInstr *> ClauseContent;
 222     I++;
 223     for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
 224       if (IsTrivialInst(I)) {
 225         ++I;
 226         continue;
 227       }
 228       if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
 229         break;
 230       std::vector<int64_t> Literals;
 231       if (I->isBundle()) {
 232         MachineInstr *DeleteMI = I;
 233         MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
 234         while (++BI != E && BI->isBundledWithPred()) {
 235           BI->unbundleFromPred();
 236           for (unsigned i = 0, e = BI->getNumOperands(); i != e; ++i) {
 237             MachineOperand &MO = BI->getOperand(i);
 238             if (MO.isReg() && MO.isInternalRead())
 239               MO.setIsInternalRead(false);
 240           }
 241           getLiteral(BI, Literals);
 242           ClauseContent.push_back(BI);
 243         }
 244         I = BI;
 245         DeleteMI->eraseFromParent();
 246       } else {
 247         getLiteral(I, Literals);
 248         ClauseContent.push_back(I);
 249         I++;
 250       }
 251       for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 252         unsigned literal0 = Literals[i];
 253         unsigned literal2 = (i + 1 < e)?Literals[i + 1]:0;
 254         MachineInstr *MILit = BuildMI(MBB, I, I->getDebugLoc(),
 255             TII->get(AMDGPU::LITERALS))
 256             .addImm(literal0)
 257             .addImm(literal2);
 258         ClauseContent.push_back(MILit);
 259       }
 260     }
 261     ClauseHead->getOperand(7).setImm(ClauseContent.size() - 1);
 262     return ClauseFile(ClauseHead, ClauseContent);
 263   }
 264
 265   void
 266   EmitFetchClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 267       unsigned &CfCount) {
 268     CounterPropagateAddr(Clause.first, CfCount);
 269     MachineBasicBlock *BB = Clause.first->getParent();
 270     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::FETCH_CLAUSE))
 271         .addImm(CfCount);
 272     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 273       BB->splice(InsertPos, BB, Clause.second[i]);
 274     }
 275     CfCount += 2 * Clause.second.size();
 276   }
 277
 278   void
 279   EmitALUClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 280       unsigned &CfCount) {
 281     CounterPropagateAddr(Clause.first, CfCount);
 282     MachineBasicBlock *BB = Clause.first->getParent();
 283     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::ALU_CLAUSE))
 284         .addImm(CfCount);
 285     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 286       BB->splice(InsertPos, BB, Clause.second[i]);
 287     }
 288     CfCount += Clause.second.size();
 289   }
 290
 291   void CounterPropagateAddr(MachineInstr *MI, unsigned Addr) const {
 292     MI->getOperand(0).setImm(Addr + MI->getOperand(0).getImm());
 293   }
 294   void CounterPropagateAddr(std::set<MachineInstr *> MIs, unsigned Addr)
 295       const {
 296     for (std::set<MachineInstr *>::iterator It = MIs.begin(), E = MIs.end();
 297         It != E; ++It) {
 298       MachineInstr *MI = *It;
 299       CounterPropagateAddr(MI, Addr);
 300     }
 301   }
 302
 303   unsigned getHWStackSize(unsigned StackSubEntry, bool hasPush) const {
 304     switch (ST.getGeneration()) {
 305     case AMDGPUSubtarget::R600:
 306     case AMDGPUSubtarget::R700:
 307       if (hasPush)
 308         StackSubEntry += 2;
 309       break;
 310     case AMDGPUSubtarget::EVERGREEN:
 311       if (hasPush)
 312         StackSubEntry ++;
 313     case AMDGPUSubtarget::NORTHERN_ISLANDS:
 314       StackSubEntry += 2;
 315       break;
 316     default: llvm_unreachable("Not a VLIW4/VLIW5 GPU");
 317     }
 318     return (StackSubEntry + 3)/4; // Need ceil value of StackSubEntry/4
 319   }
 320
 321 public:
 322   R600ControlFlowFinalizer(TargetMachine &tm) : MachineFunctionPass(ID),
 323     TII (0), TRI(0),
 324     ST(tm.getSubtarget<AMDGPUSubtarget>()) {
 325       const AMDGPUSubtarget &ST = tm.getSubtarget<AMDGPUSubtarget>();
 326       MaxFetchInst = ST.getTexVTXClauseSize();
 327   }
 328
 329   virtual bool runOnMachineFunction(MachineFunction &MF) {
 330     TII=static_cast<const R600InstrInfo *>(MF.getTarget().getInstrInfo());
 331     TRI=static_cast<const R600RegisterInfo *>(MF.getTarget().getRegisterInfo());
 332
 333     unsigned MaxStack = 0;
 334     unsigned CurrentStack = 0;
 335     bool HasPush = false;
 336     for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
 337         ++MB) {
 338       MachineBasicBlock &MBB = *MB;
 339       unsigned CfCount = 0;
 340       std::vector<std::pair<unsigned, std::set<MachineInstr *> > > LoopStack;
 341       std::vector<MachineInstr * > IfThenElseStack;
 342       R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
 343       if (MFI->ShaderType == 1) {
 344         BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
 345             getHWInstrDesc(CF_CALL_FS));
 346         CfCount++;
 347         MaxStack = 1;
 348       }
 349       std::vector<ClauseFile> FetchClauses, AluClauses;
 350       for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
 351           I != E;) {
 352         if (TII->usesTextureCache(I) || TII->usesVertexCache(I)) {
 353           DEBUG(dbgs() << CfCount << ":"; I->dump(););
 354           FetchClauses.push_back(MakeFetchClause(MBB, I));
 355           CfCount++;
 356           continue;
 357         }
 358
 359         MachineBasicBlock::iterator MI = I;
 360         I++;
 361         switch (MI->getOpcode()) {
 362         case AMDGPU::CF_ALU_PUSH_BEFORE:
 363           CurrentStack++;
 364           MaxStack = std::max(MaxStack, CurrentStack);
 365           HasPush = true;
 366         case AMDGPU::CF_ALU:
 367           I = MI;
 368           AluClauses.push_back(MakeALUClause(MBB, I));
 369         case AMDGPU::EG_ExportBuf:
 370         case AMDGPU::EG_ExportSwz:
 371         case AMDGPU::R600_ExportBuf:
 372         case AMDGPU::R600_ExportSwz:
 373         case AMDGPU::RAT_WRITE_CACHELESS_32_eg:
 374         case AMDGPU::RAT_WRITE_CACHELESS_128_eg:
 375           DEBUG(dbgs() << CfCount << ":"; MI->dump(););
 376           CfCount++;
 377           break;
 378         case AMDGPU::WHILELOOP: {
 379           CurrentStack+=4;
 380           MaxStack = std::max(MaxStack, CurrentStack);
 381           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 382               getHWInstrDesc(CF_WHILE_LOOP))
 383               .addImm(1);
 384           std::pair<unsigned, std::set<MachineInstr *> > Pair(CfCount,
 385               std::set<MachineInstr *>());
 386           Pair.second.insert(MIb);
 387           LoopStack.push_back(Pair);
 388           MI->eraseFromParent();
 389           CfCount++;
 390           break;
 391         }
 392         case AMDGPU::ENDLOOP: {
 393           CurrentStack-=4;
 394           std::pair<unsigned, std::set<MachineInstr *> > Pair =
 395               LoopStack.back();
 396           LoopStack.pop_back();
 397           CounterPropagateAddr(Pair.second, CfCount);
 398           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
 399               .addImm(Pair.first + 1);
 400           MI->eraseFromParent();
 401           CfCount++;
 402           break;
 403         }
 404         case AMDGPU::IF_PREDICATE_SET: {
 405           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 406               getHWInstrDesc(CF_JUMP))
 407               .addImm(0)
 408               .addImm(0);
 409           IfThenElseStack.push_back(MIb);
 410           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 411           MI->eraseFromParent();
 412           CfCount++;
 413           break;
 414         }
 415         case AMDGPU::ELSE: {
 416           MachineInstr * JumpInst = IfThenElseStack.back();
 417           IfThenElseStack.pop_back();
 418           CounterPropagateAddr(JumpInst, CfCount);
 419           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 420               getHWInstrDesc(CF_ELSE))
 421               .addImm(0)
 422               .addImm(1);
 423           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 424           IfThenElseStack.push_back(MIb);
 425           MI->eraseFromParent();
 426           CfCount++;
 427           break;
 428         }
 429         case AMDGPU::ENDIF: {
 430           CurrentStack--;
 431           MachineInstr *IfOrElseInst = IfThenElseStack.back();
 432           IfThenElseStack.pop_back();
 433           CounterPropagateAddr(IfOrElseInst, CfCount + 1);
 434           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 435               getHWInstrDesc(CF_POP))
 436               .addImm(CfCount + 1)
 437               .addImm(1);
 438           (void)MIb;
 439           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 440           MI->eraseFromParent();
 441           CfCount++;
 442           break;
 443         }
 444         case AMDGPU::PREDICATED_BREAK: {
 445           CurrentStack--;
 446           CfCount += 3;
 447           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_JUMP))
 448               .addImm(CfCount)
 449               .addImm(1);
 450           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 451               getHWInstrDesc(CF_LOOP_BREAK))
 452               .addImm(0);
 453           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_POP))
 454               .addImm(CfCount)
 455               .addImm(1);
 456           LoopStack.back().second.insert(MIb);
 457           MI->eraseFromParent();
 458           break;
 459         }
 460         case AMDGPU::CONTINUE: {
 461           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 462               getHWInstrDesc(CF_LOOP_CONTINUE))
 463               .addImm(0);
 464           LoopStack.back().second.insert(MIb);
 465           MI->eraseFromParent();
 466           CfCount++;
 467           break;
 468         }
 469         case AMDGPU::RETURN: {
 470           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END));
 471           CfCount++;
 472           MI->eraseFromParent();
 473           if (CfCount % 2) {
 474             BuildMI(MBB, I, MBB.findDebugLoc(MI), TII->get(AMDGPU::PAD));
 475             CfCount++;
 476           }
 477           for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
 478             EmitFetchClause(I, FetchClauses[i], CfCount);
 479           for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
 480             EmitALUClause(I, AluClauses[i], CfCount);
 481         }
 482         default:
 483           break;
 484         }
 485       }
 486       MFI->StackSize = getHWStackSize(MaxStack, HasPush);
 487     }
 488
 489     return false;
 490   }
 491
 492   const char *getPassName() const {
 493     return "R600 Control Flow Finalizer Pass";
 494   }
 495 };
 496
 497 char R600ControlFlowFinalizer::ID = 0;
 498
 499 } // end anonymous namespace
 500
 501
 502 llvm::FunctionPass *llvm::createR600ControlFlowFinalizer(TargetMachine &TM) {
 503   return new R600ControlFlowFinalizer(TM);
 504 }