lib/Target/R600/R600ControlFlowFinalizer.cpp

   1 //===-- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst----------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 /// \file
  11 /// This pass compute turns all control flow pseudo instructions into native one
  12 /// computing their address on the fly ; it also sets STACK_SIZE info.
  13 //===----------------------------------------------------------------------===//
  14
  15 #define DEBUG_TYPE "r600cf"
  16 #include "llvm/Support/Debug.h"
  17 #include "AMDGPU.h"
  18 #include "R600Defines.h"
  19 #include "R600InstrInfo.h"
  20 #include "R600MachineFunctionInfo.h"
  21 #include "R600RegisterInfo.h"
  22 #include "llvm/CodeGen/MachineFunctionPass.h"
  23 #include "llvm/CodeGen/MachineInstrBuilder.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/Support/raw_ostream.h"
  26
  27 using namespace llvm;
  28
  29 namespace {
  30
  31 class R600ControlFlowFinalizer : public MachineFunctionPass {
  32
  33 private:
  34   typedef std::pair<MachineInstr *, std::vector<MachineInstr *> > ClauseFile;
  35
  36   enum ControlFlowInstruction {
  37     CF_TC,
  38     CF_VC,
  39     CF_CALL_FS,
  40     CF_WHILE_LOOP,
  41     CF_END_LOOP,
  42     CF_LOOP_BREAK,
  43     CF_LOOP_CONTINUE,
  44     CF_JUMP,
  45     CF_ELSE,
  46     CF_POP,
  47     CF_END
  48   };
  49
  50   static char ID;
  51   const R600InstrInfo *TII;
  52   const R600RegisterInfo *TRI;
  53   unsigned MaxFetchInst;
  54   const AMDGPUSubtarget &ST;
  55
  56   bool IsTrivialInst(MachineInstr *MI) const {
  57     switch (MI->getOpcode()) {
  58     case AMDGPU::KILL:
  59     case AMDGPU::RETURN:
  60       return true;
  61     default:
  62       return false;
  63     }
  64   }
  65
  66   const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
  67     unsigned Opcode = 0;
  68     bool isEg = (ST.getGeneration() >= AMDGPUSubtarget::EVERGREEN);
  69     switch (CFI) {
  70     case CF_TC:
  71       Opcode = isEg ? AMDGPU::CF_TC_EG : AMDGPU::CF_TC_R600;
  72       break;
  73     case CF_VC:
  74       Opcode = isEg ? AMDGPU::CF_VC_EG : AMDGPU::CF_VC_R600;
  75       break;
  76     case CF_CALL_FS:
  77       Opcode = isEg ? AMDGPU::CF_CALL_FS_EG : AMDGPU::CF_CALL_FS_R600;
  78       break;
  79     case CF_WHILE_LOOP:
  80       Opcode = isEg ? AMDGPU::WHILE_LOOP_EG : AMDGPU::WHILE_LOOP_R600;
  81       break;
  82     case CF_END_LOOP:
  83       Opcode = isEg ? AMDGPU::END_LOOP_EG : AMDGPU::END_LOOP_R600;
  84       break;
  85     case CF_LOOP_BREAK:
  86       Opcode = isEg ? AMDGPU::LOOP_BREAK_EG : AMDGPU::LOOP_BREAK_R600;
  87       break;
  88     case CF_LOOP_CONTINUE:
  89       Opcode = isEg ? AMDGPU::CF_CONTINUE_EG : AMDGPU::CF_CONTINUE_R600;
  90       break;
  91     case CF_JUMP:
  92       Opcode = isEg ? AMDGPU::CF_JUMP_EG : AMDGPU::CF_JUMP_R600;
  93       break;
  94     case CF_ELSE:
  95       Opcode = isEg ? AMDGPU::CF_ELSE_EG : AMDGPU::CF_ELSE_R600;
  96       break;
  97     case CF_POP:
  98       Opcode = isEg ? AMDGPU::POP_EG : AMDGPU::POP_R600;
  99       break;
 100     case CF_END:
 101       if (ST.hasCaymanISA()) {
 102         Opcode = AMDGPU::CF_END_CM;
 103         break;
 104       }
 105       Opcode = isEg ? AMDGPU::CF_END_EG : AMDGPU::CF_END_R600;
 106       break;
 107     }
 108     assert (Opcode && "No opcode selected");
 109     return TII->get(Opcode);
 110   }
 111
 112   bool isCompatibleWithClause(const MachineInstr *MI,
 113       std::set<unsigned> &DstRegs) const {
 114     unsigned DstMI, SrcMI;
 115     for (MachineInstr::const_mop_iterator I = MI->operands_begin(),
 116         E = MI->operands_end(); I != E; ++I) {
 117       const MachineOperand &MO = *I;
 118       if (!MO.isReg())
 119         continue;
 120       if (MO.isDef()) {
 121         unsigned Reg = MO.getReg();
 122         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 123           DstMI = Reg;
 124         else
 125           DstMI = TRI->getMatchingSuperReg(Reg,
 126               TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
 127               &AMDGPU::R600_Reg128RegClass);
 128       }
 129       if (MO.isUse()) {
 130         unsigned Reg = MO.getReg();
 131         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 132           SrcMI = Reg;
 133         else
 134           SrcMI = TRI->getMatchingSuperReg(Reg,
 135               TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
 136               &AMDGPU::R600_Reg128RegClass);
 137       }
 138     }
 139     if ((DstRegs.find(SrcMI) == DstRegs.end())) {
 140       DstRegs.insert(DstMI);
 141       return true;
 142     } else
 143       return false;
 144   }
 145
 146   ClauseFile
 147   MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 148       const {
 149     MachineBasicBlock::iterator ClauseHead = I;
 150     std::vector<MachineInstr *> ClauseContent;
 151     unsigned AluInstCount = 0;
 152     bool IsTex = TII->usesTextureCache(ClauseHead);
 153     std::set<unsigned> DstRegs;
 154     for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
 155       if (IsTrivialInst(I))
 156         continue;
 157       if (AluInstCount >= MaxFetchInst)
 158         break;
 159       if ((IsTex && !TII->usesTextureCache(I)) ||
 160           (!IsTex && !TII->usesVertexCache(I)))
 161         break;
 162       if (!isCompatibleWithClause(I, DstRegs))
 163         break;
 164       AluInstCount ++;
 165       ClauseContent.push_back(I);
 166     }
 167     MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
 168         getHWInstrDesc(IsTex?CF_TC:CF_VC))
 169         .addImm(0) // ADDR
 170         .addImm(AluInstCount - 1); // COUNT
 171     return ClauseFile(MIb, ClauseContent);
 172   }
 173
 174   void getLiteral(MachineInstr *MI, std::vector<int64_t> &Lits) const {
 175     unsigned LiteralRegs[] = {
 176       AMDGPU::ALU_LITERAL_X,
 177       AMDGPU::ALU_LITERAL_Y,
 178       AMDGPU::ALU_LITERAL_Z,
 179       AMDGPU::ALU_LITERAL_W
 180     };
 181     const SmallVector<std::pair<MachineOperand *, int64_t>, 3 > Srcs =
 182         TII->getSrcs(MI);
 183     for (unsigned i = 0, e = Srcs.size(); i < e; ++i) {
 184       if (Srcs[i].first->getReg() != AMDGPU::ALU_LITERAL_X)
 185         continue;
 186       int64_t Imm = Srcs[i].second;
 187       std::vector<int64_t>::iterator It =
 188           std::find(Lits.begin(), Lits.end(), Imm);
 189       if (It != Lits.end()) {
 190         unsigned Index = It - Lits.begin();
 191         Srcs[i].first->setReg(LiteralRegs[Index]);
 192       } else {
 193         assert(Lits.size() < 4 && "Too many literals in Instruction Group");
 194         Srcs[i].first->setReg(LiteralRegs[Lits.size()]);
 195         Lits.push_back(Imm);
 196       }
 197     }
 198   }
 199
 200   MachineBasicBlock::iterator insertLiterals(
 201       MachineBasicBlock::iterator InsertPos,
 202       const std::vector<unsigned> &Literals) const {
 203     MachineBasicBlock *MBB = InsertPos->getParent();
 204     for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 205       unsigned LiteralPair0 = Literals[i];
 206       unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
 207       InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
 208           TII->get(AMDGPU::LITERALS))
 209           .addImm(LiteralPair0)
 210           .addImm(LiteralPair1);
 211     }
 212     return InsertPos;
 213   }
 214
 215   ClauseFile
 216   MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 217       const {
 218     MachineBasicBlock::iterator ClauseHead = I;
 219     std::vector<MachineInstr *> ClauseContent;
 220     I++;
 221     for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
 222       if (IsTrivialInst(I)) {
 223         ++I;
 224         continue;
 225       }
 226       if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
 227         break;
 228       std::vector<int64_t> Literals;
 229       if (I->isBundle()) {
 230         MachineInstr *DeleteMI = I;
 231         MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
 232         while (++BI != E && BI->isBundledWithPred()) {
 233           BI->unbundleFromPred();
 234           for (unsigned i = 0, e = BI->getNumOperands(); i != e; ++i) {
 235             MachineOperand &MO = BI->getOperand(i);
 236             if (MO.isReg() && MO.isInternalRead())
 237               MO.setIsInternalRead(false);
 238           }
 239           getLiteral(BI, Literals);
 240           ClauseContent.push_back(BI);
 241         }
 242         I = BI;
 243         DeleteMI->eraseFromParent();
 244       } else {
 245         getLiteral(I, Literals);
 246         ClauseContent.push_back(I);
 247         I++;
 248       }
 249       for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 250         unsigned literal0 = Literals[i];
 251         unsigned literal2 = (i + 1 < e)?Literals[i + 1]:0;
 252         MachineInstr *MILit = BuildMI(MBB, I, I->getDebugLoc(),
 253             TII->get(AMDGPU::LITERALS))
 254             .addImm(literal0)
 255             .addImm(literal2);
 256         ClauseContent.push_back(MILit);
 257       }
 258     }
 259     ClauseHead->getOperand(7).setImm(ClauseContent.size() - 1);
 260     return ClauseFile(ClauseHead, ClauseContent);
 261   }
 262
 263   void
 264   EmitFetchClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 265       unsigned &CfCount) {
 266     CounterPropagateAddr(Clause.first, CfCount);
 267     MachineBasicBlock *BB = Clause.first->getParent();
 268     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::FETCH_CLAUSE))
 269         .addImm(CfCount);
 270     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 271       BB->splice(InsertPos, BB, Clause.second[i]);
 272     }
 273     CfCount += 2 * Clause.second.size();
 274   }
 275
 276   void
 277   EmitALUClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 278       unsigned &CfCount) {
 279     CounterPropagateAddr(Clause.first, CfCount);
 280     MachineBasicBlock *BB = Clause.first->getParent();
 281     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::ALU_CLAUSE))
 282         .addImm(CfCount);
 283     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 284       BB->splice(InsertPos, BB, Clause.second[i]);
 285     }
 286     CfCount += Clause.second.size();
 287   }
 288
 289   void CounterPropagateAddr(MachineInstr *MI, unsigned Addr) const {
 290     MI->getOperand(0).setImm(Addr + MI->getOperand(0).getImm());
 291   }
 292   void CounterPropagateAddr(std::set<MachineInstr *> MIs, unsigned Addr)
 293       const {
 294     for (std::set<MachineInstr *>::iterator It = MIs.begin(), E = MIs.end();
 295         It != E; ++It) {
 296       MachineInstr *MI = *It;
 297       CounterPropagateAddr(MI, Addr);
 298     }
 299   }
 300
 301   unsigned getHWStackSize(unsigned StackSubEntry, bool hasPush) const {
 302     switch (ST.getGeneration()) {
 303     case AMDGPUSubtarget::R600:
 304     case AMDGPUSubtarget::R700:
 305       if (hasPush)
 306         StackSubEntry += 2;
 307       break;
 308     case AMDGPUSubtarget::EVERGREEN:
 309       if (hasPush)
 310         StackSubEntry ++;
 311     case AMDGPUSubtarget::NORTHERN_ISLANDS:
 312       StackSubEntry += 2;
 313       break;
 314     default: llvm_unreachable("Not a VLIW4/VLIW5 GPU");
 315     }
 316     return (StackSubEntry + 3)/4; // Need ceil value of StackSubEntry/4
 317   }
 318
 319 public:
 320   R600ControlFlowFinalizer(TargetMachine &tm) : MachineFunctionPass(ID),
 321     TII (0), TRI(0),
 322     ST(tm.getSubtarget<AMDGPUSubtarget>()) {
 323       const AMDGPUSubtarget &ST = tm.getSubtarget<AMDGPUSubtarget>();
 324       MaxFetchInst = ST.getTexVTXClauseSize();
 325   }
 326
 327   virtual bool runOnMachineFunction(MachineFunction &MF) {
 328     TII=static_cast<const R600InstrInfo *>(MF.getTarget().getInstrInfo());
 329     TRI=static_cast<const R600RegisterInfo *>(MF.getTarget().getRegisterInfo());
 330
 331     unsigned MaxStack = 0;
 332     unsigned CurrentStack = 0;
 333     bool HasPush = false;
 334     for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
 335         ++MB) {
 336       MachineBasicBlock &MBB = *MB;
 337       unsigned CfCount = 0;
 338       std::vector<std::pair<unsigned, std::set<MachineInstr *> > > LoopStack;
 339       std::vector<MachineInstr * > IfThenElseStack;
 340       R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
 341       if (MFI->ShaderType == 1) {
 342         BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
 343             getHWInstrDesc(CF_CALL_FS));
 344         CfCount++;
 345         MaxStack = 1;
 346       }
 347       std::vector<ClauseFile> FetchClauses, AluClauses;
 348       for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
 349           I != E;) {
 350         if (TII->usesTextureCache(I) || TII->usesVertexCache(I)) {
 351           DEBUG(dbgs() << CfCount << ":"; I->dump(););
 352           FetchClauses.push_back(MakeFetchClause(MBB, I));
 353           CfCount++;
 354           continue;
 355         }
 356
 357         MachineBasicBlock::iterator MI = I;
 358         I++;
 359         switch (MI->getOpcode()) {
 360         case AMDGPU::CF_ALU_PUSH_BEFORE:
 361           CurrentStack++;
 362           MaxStack = std::max(MaxStack, CurrentStack);
 363           HasPush = true;
 364         case AMDGPU::CF_ALU:
 365           I = MI;
 366           AluClauses.push_back(MakeALUClause(MBB, I));
 367         case AMDGPU::EG_ExportBuf:
 368         case AMDGPU::EG_ExportSwz:
 369         case AMDGPU::R600_ExportBuf:
 370         case AMDGPU::R600_ExportSwz:
 371         case AMDGPU::RAT_WRITE_CACHELESS_32_eg:
 372         case AMDGPU::RAT_WRITE_CACHELESS_128_eg:
 373           DEBUG(dbgs() << CfCount << ":"; MI->dump(););
 374           CfCount++;
 375           break;
 376         case AMDGPU::WHILELOOP: {
 377           CurrentStack+=4;
 378           MaxStack = std::max(MaxStack, CurrentStack);
 379           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 380               getHWInstrDesc(CF_WHILE_LOOP))
 381               .addImm(1);
 382           std::pair<unsigned, std::set<MachineInstr *> > Pair(CfCount,
 383               std::set<MachineInstr *>());
 384           Pair.second.insert(MIb);
 385           LoopStack.push_back(Pair);
 386           MI->eraseFromParent();
 387           CfCount++;
 388           break;
 389         }
 390         case AMDGPU::ENDLOOP: {
 391           CurrentStack-=4;
 392           std::pair<unsigned, std::set<MachineInstr *> > Pair =
 393               LoopStack.back();
 394           LoopStack.pop_back();
 395           CounterPropagateAddr(Pair.second, CfCount);
 396           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
 397               .addImm(Pair.first + 1);
 398           MI->eraseFromParent();
 399           CfCount++;
 400           break;
 401         }
 402         case AMDGPU::IF_PREDICATE_SET: {
 403           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 404               getHWInstrDesc(CF_JUMP))
 405               .addImm(0)
 406               .addImm(0);
 407           IfThenElseStack.push_back(MIb);
 408           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 409           MI->eraseFromParent();
 410           CfCount++;
 411           break;
 412         }
 413         case AMDGPU::ELSE: {
 414           MachineInstr * JumpInst = IfThenElseStack.back();
 415           IfThenElseStack.pop_back();
 416           CounterPropagateAddr(JumpInst, CfCount);
 417           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 418               getHWInstrDesc(CF_ELSE))
 419               .addImm(0)
 420               .addImm(1);
 421           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 422           IfThenElseStack.push_back(MIb);
 423           MI->eraseFromParent();
 424           CfCount++;
 425           break;
 426         }
 427         case AMDGPU::ENDIF: {
 428           CurrentStack--;
 429           MachineInstr *IfOrElseInst = IfThenElseStack.back();
 430           IfThenElseStack.pop_back();
 431           CounterPropagateAddr(IfOrElseInst, CfCount + 1);
 432           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 433               getHWInstrDesc(CF_POP))
 434               .addImm(CfCount + 1)
 435               .addImm(1);
 436           (void)MIb;
 437           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 438           MI->eraseFromParent();
 439           CfCount++;
 440           break;
 441         }
 442         case AMDGPU::PREDICATED_BREAK: {
 443           CurrentStack--;
 444           CfCount += 3;
 445           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_JUMP))
 446               .addImm(CfCount)
 447               .addImm(1);
 448           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 449               getHWInstrDesc(CF_LOOP_BREAK))
 450               .addImm(0);
 451           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_POP))
 452               .addImm(CfCount)
 453               .addImm(1);
 454           LoopStack.back().second.insert(MIb);
 455           MI->eraseFromParent();
 456           break;
 457         }
 458         case AMDGPU::CONTINUE: {
 459           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 460               getHWInstrDesc(CF_LOOP_CONTINUE))
 461               .addImm(0);
 462           LoopStack.back().second.insert(MIb);
 463           MI->eraseFromParent();
 464           CfCount++;
 465           break;
 466         }
 467         case AMDGPU::RETURN: {
 468           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END));
 469           CfCount++;
 470           MI->eraseFromParent();
 471           if (CfCount % 2) {
 472             BuildMI(MBB, I, MBB.findDebugLoc(MI), TII->get(AMDGPU::PAD));
 473             CfCount++;
 474           }
 475           for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
 476             EmitFetchClause(I, FetchClauses[i], CfCount);
 477           for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
 478             EmitALUClause(I, AluClauses[i], CfCount);
 479         }
 480         default:
 481           break;
 482         }
 483       }
 484       MFI->StackSize = getHWStackSize(MaxStack, HasPush);
 485     }
 486
 487     return false;
 488   }
 489
 490   const char *getPassName() const {
 491     return "R600 Control Flow Finalizer Pass";
 492   }
 493 };
 494
 495 char R600ControlFlowFinalizer::ID = 0;
 496
 497 } // end anonymous namespace
 498
 499
 500 llvm::FunctionPass *llvm::createR600ControlFlowFinalizer(TargetMachine &TM) {
 501   return new R600ControlFlowFinalizer(TM);
 502 }