lib/Target/R600/R600ControlFlowFinalizer.cpp

   1 //===-- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst----------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 /// \file
  11 /// This pass compute turns all control flow pseudo instructions into native one
  12 /// computing their address on the fly ; it also sets STACK_SIZE info.
  13 //===----------------------------------------------------------------------===//
  14
  15 #define DEBUG_TYPE "r600cf"
  16 #include "llvm/Support/Debug.h"
  17 #include "AMDGPU.h"
  18 #include "R600Defines.h"
  19 #include "R600InstrInfo.h"
  20 #include "R600MachineFunctionInfo.h"
  21 #include "R600RegisterInfo.h"
  22 #include "llvm/CodeGen/MachineFunctionPass.h"
  23 #include "llvm/CodeGen/MachineInstrBuilder.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/Support/raw_ostream.h"
  26
  27 using namespace llvm;
  28
  29 namespace {
  30
  31 class R600ControlFlowFinalizer : public MachineFunctionPass {
  32
  33 private:
  34   typedef std::pair<MachineInstr *, std::vector<MachineInstr *> > ClauseFile;
  35
  36   enum ControlFlowInstruction {
  37     CF_TC,
  38     CF_VC,
  39     CF_CALL_FS,
  40     CF_WHILE_LOOP,
  41     CF_END_LOOP,
  42     CF_LOOP_BREAK,
  43     CF_LOOP_CONTINUE,
  44     CF_JUMP,
  45     CF_ELSE,
  46     CF_POP,
  47     CF_END
  48   };
  49
  50   static char ID;
  51   const R600InstrInfo *TII;
  52   const R600RegisterInfo &TRI;
  53   unsigned MaxFetchInst;
  54   const AMDGPUSubtarget &ST;
  55
  56   bool IsTrivialInst(MachineInstr *MI) const {
  57     switch (MI->getOpcode()) {
  58     case AMDGPU::KILL:
  59     case AMDGPU::RETURN:
  60       return true;
  61     default:
  62       return false;
  63     }
  64   }
  65
  66   const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
  67     unsigned Opcode = 0;
  68     bool isEg = (ST.device()->getGeneration() >= AMDGPUDeviceInfo::HD5XXX);
  69     switch (CFI) {
  70     case CF_TC:
  71       Opcode = isEg ? AMDGPU::CF_TC_EG : AMDGPU::CF_TC_R600;
  72       break;
  73     case CF_VC:
  74       Opcode = isEg ? AMDGPU::CF_VC_EG : AMDGPU::CF_VC_R600;
  75       break;
  76     case CF_CALL_FS:
  77       Opcode = isEg ? AMDGPU::CF_CALL_FS_EG : AMDGPU::CF_CALL_FS_R600;
  78       break;
  79     case CF_WHILE_LOOP:
  80       Opcode = isEg ? AMDGPU::WHILE_LOOP_EG : AMDGPU::WHILE_LOOP_R600;
  81       break;
  82     case CF_END_LOOP:
  83       Opcode = isEg ? AMDGPU::END_LOOP_EG : AMDGPU::END_LOOP_R600;
  84       break;
  85     case CF_LOOP_BREAK:
  86       Opcode = isEg ? AMDGPU::LOOP_BREAK_EG : AMDGPU::LOOP_BREAK_R600;
  87       break;
  88     case CF_LOOP_CONTINUE:
  89       Opcode = isEg ? AMDGPU::CF_CONTINUE_EG : AMDGPU::CF_CONTINUE_R600;
  90       break;
  91     case CF_JUMP:
  92       Opcode = isEg ? AMDGPU::CF_JUMP_EG : AMDGPU::CF_JUMP_R600;
  93       break;
  94     case CF_ELSE:
  95       Opcode = isEg ? AMDGPU::CF_ELSE_EG : AMDGPU::CF_ELSE_R600;
  96       break;
  97     case CF_POP:
  98       Opcode = isEg ? AMDGPU::POP_EG : AMDGPU::POP_R600;
  99       break;
 100     case CF_END:
 101       if (ST.device()->getDeviceFlag() == OCL_DEVICE_CAYMAN) {
 102         Opcode = AMDGPU::CF_END_CM;
 103         break;
 104       }
 105       Opcode = isEg ? AMDGPU::CF_END_EG : AMDGPU::CF_END_R600;
 106       break;
 107     }
 108     assert (Opcode && "No opcode selected");
 109     return TII->get(Opcode);
 110   }
 111
 112   bool isCompatibleWithClause(const MachineInstr *MI,
 113   std::set<unsigned> &DstRegs, std::set<unsigned> &SrcRegs) const {
 114     unsigned DstMI, SrcMI;
 115     for (MachineInstr::const_mop_iterator I = MI->operands_begin(),
 116         E = MI->operands_end(); I != E; ++I) {
 117       const MachineOperand &MO = *I;
 118       if (!MO.isReg())
 119         continue;
 120       if (MO.isDef())
 121         DstMI = MO.getReg();
 122       if (MO.isUse()) {
 123         unsigned Reg = MO.getReg();
 124         if (AMDGPU::R600_Reg128RegClass.contains(Reg))
 125           SrcMI = Reg;
 126         else
 127           SrcMI = TRI.getMatchingSuperReg(Reg,
 128               TRI.getSubRegFromChannel(TRI.getHWRegChan(Reg)),
 129               &AMDGPU::R600_Reg128RegClass);
 130       }
 131     }
 132     if ((DstRegs.find(SrcMI) == DstRegs.end()) &&
 133         (SrcRegs.find(DstMI) == SrcRegs.end())) {
 134       SrcRegs.insert(SrcMI);
 135       DstRegs.insert(DstMI);
 136       return true;
 137     } else
 138       return false;
 139   }
 140
 141   ClauseFile
 142   MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 143       const {
 144     MachineBasicBlock::iterator ClauseHead = I;
 145     std::vector<MachineInstr *> ClauseContent;
 146     unsigned AluInstCount = 0;
 147     bool IsTex = TII->usesTextureCache(ClauseHead);
 148     std::set<unsigned> DstRegs, SrcRegs;
 149     for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
 150       if (IsTrivialInst(I))
 151         continue;
 152       if (AluInstCount >= MaxFetchInst)
 153         break;
 154       if ((IsTex && !TII->usesTextureCache(I)) ||
 155           (!IsTex && !TII->usesVertexCache(I)))
 156         break;
 157       if (!isCompatibleWithClause(I, DstRegs, SrcRegs))
 158         break;
 159       AluInstCount ++;
 160       ClauseContent.push_back(I);
 161     }
 162     MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
 163         getHWInstrDesc(IsTex?CF_TC:CF_VC))
 164         .addImm(0) // ADDR
 165         .addImm(AluInstCount - 1); // COUNT
 166     return ClauseFile(MIb, ClauseContent);
 167   }
 168
 169   void getLiteral(MachineInstr *MI, std::vector<int64_t> &Lits) const {
 170     unsigned LiteralRegs[] = {
 171       AMDGPU::ALU_LITERAL_X,
 172       AMDGPU::ALU_LITERAL_Y,
 173       AMDGPU::ALU_LITERAL_Z,
 174       AMDGPU::ALU_LITERAL_W
 175     };
 176     const SmallVector<std::pair<MachineOperand *, int64_t>, 3 > Srcs =
 177         TII->getSrcs(MI);
 178     for (unsigned i = 0, e = Srcs.size(); i < e; ++i) {
 179       if (Srcs[i].first->getReg() != AMDGPU::ALU_LITERAL_X)
 180         continue;
 181       int64_t Imm = Srcs[i].second;
 182       std::vector<int64_t>::iterator It =
 183           std::find(Lits.begin(), Lits.end(), Imm);
 184       if (It != Lits.end()) {
 185         unsigned Index = It - Lits.begin();
 186         Srcs[i].first->setReg(LiteralRegs[Index]);
 187       } else {
 188         assert(Lits.size() < 4 && "Too many literals in Instruction Group");
 189         Srcs[i].first->setReg(LiteralRegs[Lits.size()]);
 190         Lits.push_back(Imm);
 191       }
 192     }
 193   }
 194
 195   MachineBasicBlock::iterator insertLiterals(
 196       MachineBasicBlock::iterator InsertPos,
 197       const std::vector<unsigned> &Literals) const {
 198     MachineBasicBlock *MBB = InsertPos->getParent();
 199     for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 200       unsigned LiteralPair0 = Literals[i];
 201       unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
 202       InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
 203           TII->get(AMDGPU::LITERALS))
 204           .addImm(LiteralPair0)
 205           .addImm(LiteralPair1);
 206     }
 207     return InsertPos;
 208   }
 209
 210   ClauseFile
 211   MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
 212       const {
 213     MachineBasicBlock::iterator ClauseHead = I;
 214     std::vector<MachineInstr *> ClauseContent;
 215     I++;
 216     for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
 217       if (IsTrivialInst(I)) {
 218         ++I;
 219         continue;
 220       }
 221       if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
 222         break;
 223       std::vector<int64_t> Literals;
 224       if (I->isBundle()) {
 225         MachineInstr *DeleteMI = I;
 226         MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
 227         while (++BI != E && BI->isBundledWithPred()) {
 228           BI->unbundleFromPred();
 229           for (unsigned i = 0, e = BI->getNumOperands(); i != e; ++i) {
 230             MachineOperand &MO = BI->getOperand(i);
 231             if (MO.isReg() && MO.isInternalRead())
 232               MO.setIsInternalRead(false);
 233           }
 234           getLiteral(BI, Literals);
 235           ClauseContent.push_back(BI);
 236         }
 237         I = BI;
 238         DeleteMI->eraseFromParent();
 239       } else {
 240         getLiteral(I, Literals);
 241         ClauseContent.push_back(I);
 242         I++;
 243       }
 244       for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
 245         unsigned literal0 = Literals[i];
 246         unsigned literal2 = (i + 1 < e)?Literals[i + 1]:0;
 247         MachineInstr *MILit = BuildMI(MBB, I, I->getDebugLoc(),
 248             TII->get(AMDGPU::LITERALS))
 249             .addImm(literal0)
 250             .addImm(literal2);
 251         ClauseContent.push_back(MILit);
 252       }
 253     }
 254     ClauseHead->getOperand(7).setImm(ClauseContent.size() - 1);
 255     return ClauseFile(ClauseHead, ClauseContent);
 256   }
 257
 258   void
 259   EmitFetchClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 260       unsigned &CfCount) {
 261     CounterPropagateAddr(Clause.first, CfCount);
 262     MachineBasicBlock *BB = Clause.first->getParent();
 263     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::FETCH_CLAUSE))
 264         .addImm(CfCount);
 265     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 266       BB->splice(InsertPos, BB, Clause.second[i]);
 267     }
 268     CfCount += 2 * Clause.second.size();
 269   }
 270
 271   void
 272   EmitALUClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
 273       unsigned &CfCount) {
 274     CounterPropagateAddr(Clause.first, CfCount);
 275     MachineBasicBlock *BB = Clause.first->getParent();
 276     BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::ALU_CLAUSE))
 277         .addImm(CfCount);
 278     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
 279       BB->splice(InsertPos, BB, Clause.second[i]);
 280     }
 281     CfCount += Clause.second.size();
 282   }
 283
 284   void CounterPropagateAddr(MachineInstr *MI, unsigned Addr) const {
 285     MI->getOperand(0).setImm(Addr + MI->getOperand(0).getImm());
 286   }
 287   void CounterPropagateAddr(std::set<MachineInstr *> MIs, unsigned Addr)
 288       const {
 289     for (std::set<MachineInstr *>::iterator It = MIs.begin(), E = MIs.end();
 290         It != E; ++It) {
 291       MachineInstr *MI = *It;
 292       CounterPropagateAddr(MI, Addr);
 293     }
 294   }
 295
 296   unsigned getHWStackSize(unsigned StackSubEntry, bool hasPush) const {
 297     switch (ST.device()->getGeneration()) {
 298     case AMDGPUDeviceInfo::HD4XXX:
 299       if (hasPush)
 300         StackSubEntry += 2;
 301       break;
 302     case AMDGPUDeviceInfo::HD5XXX:
 303       if (hasPush)
 304         StackSubEntry ++;
 305     case AMDGPUDeviceInfo::HD6XXX:
 306       StackSubEntry += 2;
 307       break;
 308     }
 309     return (StackSubEntry + 3)/4; // Need ceil value of StackSubEntry/4
 310   }
 311
 312 public:
 313   R600ControlFlowFinalizer(TargetMachine &tm) : MachineFunctionPass(ID),
 314     TII (static_cast<const R600InstrInfo *>(tm.getInstrInfo())),
 315     TRI(TII->getRegisterInfo()),
 316     ST(tm.getSubtarget<AMDGPUSubtarget>()) {
 317       const AMDGPUSubtarget &ST = tm.getSubtarget<AMDGPUSubtarget>();
 318       MaxFetchInst = ST.getTexVTXClauseSize();
 319   }
 320
 321   virtual bool runOnMachineFunction(MachineFunction &MF) {
 322     unsigned MaxStack = 0;
 323     unsigned CurrentStack = 0;
 324     bool HasPush = false;
 325     for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
 326         ++MB) {
 327       MachineBasicBlock &MBB = *MB;
 328       unsigned CfCount = 0;
 329       std::vector<std::pair<unsigned, std::set<MachineInstr *> > > LoopStack;
 330       std::vector<MachineInstr * > IfThenElseStack;
 331       R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
 332       if (MFI->ShaderType == 1) {
 333         BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
 334             getHWInstrDesc(CF_CALL_FS));
 335         CfCount++;
 336       }
 337       std::vector<ClauseFile> FetchClauses, AluClauses;
 338       for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
 339           I != E;) {
 340         if (TII->usesTextureCache(I) || TII->usesVertexCache(I)) {
 341           DEBUG(dbgs() << CfCount << ":"; I->dump(););
 342           FetchClauses.push_back(MakeFetchClause(MBB, I));
 343           CfCount++;
 344           continue;
 345         }
 346
 347         MachineBasicBlock::iterator MI = I;
 348         I++;
 349         switch (MI->getOpcode()) {
 350         case AMDGPU::CF_ALU_PUSH_BEFORE:
 351           CurrentStack++;
 352           MaxStack = std::max(MaxStack, CurrentStack);
 353           HasPush = true;
 354         case AMDGPU::CF_ALU:
 355           I = MI;
 356           AluClauses.push_back(MakeALUClause(MBB, I));
 357         case AMDGPU::EG_ExportBuf:
 358         case AMDGPU::EG_ExportSwz:
 359         case AMDGPU::R600_ExportBuf:
 360         case AMDGPU::R600_ExportSwz:
 361         case AMDGPU::RAT_WRITE_CACHELESS_32_eg:
 362         case AMDGPU::RAT_WRITE_CACHELESS_128_eg:
 363           DEBUG(dbgs() << CfCount << ":"; MI->dump(););
 364           CfCount++;
 365           break;
 366         case AMDGPU::WHILELOOP: {
 367           CurrentStack+=4;
 368           MaxStack = std::max(MaxStack, CurrentStack);
 369           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 370               getHWInstrDesc(CF_WHILE_LOOP))
 371               .addImm(1);
 372           std::pair<unsigned, std::set<MachineInstr *> > Pair(CfCount,
 373               std::set<MachineInstr *>());
 374           Pair.second.insert(MIb);
 375           LoopStack.push_back(Pair);
 376           MI->eraseFromParent();
 377           CfCount++;
 378           break;
 379         }
 380         case AMDGPU::ENDLOOP: {
 381           CurrentStack-=4;
 382           std::pair<unsigned, std::set<MachineInstr *> > Pair =
 383               LoopStack.back();
 384           LoopStack.pop_back();
 385           CounterPropagateAddr(Pair.second, CfCount);
 386           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
 387               .addImm(Pair.first + 1);
 388           MI->eraseFromParent();
 389           CfCount++;
 390           break;
 391         }
 392         case AMDGPU::IF_PREDICATE_SET: {
 393           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 394               getHWInstrDesc(CF_JUMP))
 395               .addImm(0)
 396               .addImm(0);
 397           IfThenElseStack.push_back(MIb);
 398           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 399           MI->eraseFromParent();
 400           CfCount++;
 401           break;
 402         }
 403         case AMDGPU::ELSE: {
 404           MachineInstr * JumpInst = IfThenElseStack.back();
 405           IfThenElseStack.pop_back();
 406           CounterPropagateAddr(JumpInst, CfCount);
 407           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 408               getHWInstrDesc(CF_ELSE))
 409               .addImm(0)
 410               .addImm(1);
 411           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 412           IfThenElseStack.push_back(MIb);
 413           MI->eraseFromParent();
 414           CfCount++;
 415           break;
 416         }
 417         case AMDGPU::ENDIF: {
 418           CurrentStack--;
 419           MachineInstr *IfOrElseInst = IfThenElseStack.back();
 420           IfThenElseStack.pop_back();
 421           CounterPropagateAddr(IfOrElseInst, CfCount + 1);
 422           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 423               getHWInstrDesc(CF_POP))
 424               .addImm(CfCount + 1)
 425               .addImm(1);
 426           (void)MIb;
 427           DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
 428           MI->eraseFromParent();
 429           CfCount++;
 430           break;
 431         }
 432         case AMDGPU::PREDICATED_BREAK: {
 433           CurrentStack--;
 434           CfCount += 3;
 435           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_JUMP))
 436               .addImm(CfCount)
 437               .addImm(1);
 438           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 439               getHWInstrDesc(CF_LOOP_BREAK))
 440               .addImm(0);
 441           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_POP))
 442               .addImm(CfCount)
 443               .addImm(1);
 444           LoopStack.back().second.insert(MIb);
 445           MI->eraseFromParent();
 446           break;
 447         }
 448         case AMDGPU::CONTINUE: {
 449           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
 450               getHWInstrDesc(CF_LOOP_CONTINUE))
 451               .addImm(0);
 452           LoopStack.back().second.insert(MIb);
 453           MI->eraseFromParent();
 454           CfCount++;
 455           break;
 456         }
 457         case AMDGPU::RETURN: {
 458           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END));
 459           CfCount++;
 460           MI->eraseFromParent();
 461           if (CfCount % 2) {
 462             BuildMI(MBB, I, MBB.findDebugLoc(MI), TII->get(AMDGPU::PAD));
 463             CfCount++;
 464           }
 465           for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
 466             EmitFetchClause(I, FetchClauses[i], CfCount);
 467           for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
 468             EmitALUClause(I, AluClauses[i], CfCount);
 469         }
 470         default:
 471           break;
 472         }
 473       }
 474       MFI->StackSize = getHWStackSize(MaxStack, HasPush);
 475     }
 476
 477     return false;
 478   }
 479
 480   const char *getPassName() const {
 481     return "R600 Control Flow Finalizer Pass";
 482   }
 483 };
 484
 485 char R600ControlFlowFinalizer::ID = 0;
 486
 487 } // end anonymous namespace
 488
 489
 490 llvm::FunctionPass *llvm::createR600ControlFlowFinalizer(TargetMachine &TM) {
 491   return new R600ControlFlowFinalizer(TM);
 492 }