add some noop code to push it out of my tree.
[oota-llvm.git] / utils / TableGen / FastISelEmitter.cpp
index 7edd5fca420eeb33256b0aa2d1eb860442555b00..f589bcc9e45999e0ff1603890c452b08fc28651e 100644 (file)
@@ -7,43 +7,34 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This tablegen backend emits a "fast" instruction selector.
+// This tablegen backend emits code for use by the "fast" instruction
+// selection algorithm. See the comments at the top of
+// lib/CodeGen/SelectionDAG/FastISel.cpp for background.
 //
-// This instruction selection method is designed to emit very poor code
-// quickly. Also, it is not designed to do much lowering, so most illegal
-// types (e.g. i64 on 32-bit targets) and operations (e.g. calls) are not
-// supported and cannot easily be added. Blocks containing operations
-// that are not supported need to be handled by a more capable selector,
-// such as the SelectionDAG selector.
+// This file scans through the target's tablegen instruction-info files
+// and extracts instructions with obvious-looking patterns, and it emits
+// code to look up these instructions by type and operator.
 //
-// The intended use for "fast" instruction selection is "-O0" mode
-// compilation, where the quality of the generated code is irrelevant when
-// weighed against the speed at which the code can be generated.
-//
-// If compile time is so important, you might wonder why we don't just
-// skip codegen all-together, emit LLVM bytecode files, and execute them
-// with an interpreter. The answer is that it would complicate linking and
-// debugging, and also because that isn't how a compiler is expected to
-// work in some circles.
-//
-// If you need better generated code or more lowering than what this
-// instruction selector provides, use the SelectionDAG (DAGISel) instruction
-// selector instead. If you're looking here because SelectionDAG isn't fast
-// enough, consider looking into improving the SelectionDAG infastructure
-// instead. At the time of this writing there remain several major
-// opportunities for improvement.
-// 
 //===----------------------------------------------------------------------===//
 
 #include "FastISelEmitter.h"
 #include "Record.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/Streams.h"
 #include "llvm/ADT/VectorExtras.h"
 using namespace llvm;
 
 namespace {
 
+/// InstructionMemo - This class holds additional information about an
+/// instruction needed to emit code for it.
+///
+struct InstructionMemo {
+  std::string Name;
+  const CodeGenRegisterClass *RC;
+  unsigned char SubRegNo;
+  std::vector<std::string>* PhysRegs;
+};
+
 /// OperandsSignature - This class holds a description of a list of operand
 /// types. It has utility methods for emitting text based on the operands.
 ///
@@ -62,42 +53,79 @@ struct OperandsSignature {
   ///
   bool initialize(TreePatternNode *InstPatNode,
                   const CodeGenTarget &Target,
-                  MVT::SimpleValueType VT,
-                  const CodeGenRegisterClass *DstRC) {
+                  MVT::SimpleValueType VT) {
+    if (!InstPatNode->isLeaf() &&
+        InstPatNode->getOperator()->getName() == "imm") {
+      Operands.push_back("i");
+      return true;
+    }
+    if (!InstPatNode->isLeaf() &&
+        InstPatNode->getOperator()->getName() == "fpimm") {
+      Operands.push_back("f");
+      return true;
+    }
+    
+    const CodeGenRegisterClass *DstRC = 0;
+    
     for (unsigned i = 0, e = InstPatNode->getNumChildren(); i != e; ++i) {
       TreePatternNode *Op = InstPatNode->getChild(i);
-      if (!Op->isLeaf())
-        return false;
       // For now, filter out any operand with a predicate.
-      if (!Op->getPredicateFn().empty())
+      if (!Op->getPredicateFns().empty())
+        return false;
+      // For now, filter out any operand with multiple values.
+      if (Op->getExtTypes().size() != 1)
         return false;
+      // For now, all the operands must have the same type.
+      if (Op->getTypeNum(0) != VT)
+        return false;
+      if (!Op->isLeaf()) {
+        if (Op->getOperator()->getName() == "imm") {
+          Operands.push_back("i");
+          continue;
+        }
+        if (Op->getOperator()->getName() == "fpimm") {
+          Operands.push_back("f");
+          continue;
+        }
+        // For now, ignore other non-leaf nodes.
+        return false;
+      }
       DefInit *OpDI = dynamic_cast<DefInit*>(Op->getLeafValue());
       if (!OpDI)
         return false;
       Record *OpLeafRec = OpDI->getDef();
-      // For now, only accept register operands.
-      if (!OpLeafRec->isSubClassOf("RegisterClass"))
+      // For now, the only other thing we accept is register operands.
+
+      const CodeGenRegisterClass *RC = 0;
+      if (OpLeafRec->isSubClassOf("RegisterClass"))
+        RC = &Target.getRegisterClass(OpLeafRec);
+      else if (OpLeafRec->isSubClassOf("Register"))
+        RC = Target.getRegisterClassForRegister(OpLeafRec);
+      else
         return false;
       // For now, require the register operands' register classes to all
       // be the same.
-      const CodeGenRegisterClass *RC = &Target.getRegisterClass(OpLeafRec);
       if (!RC)
         return false;
       // For now, all the operands must have the same register class.
-      if (DstRC != RC)
-        return false;
-      // For now, all the operands must have the same type.
-      if (Op->getTypeNum(0) != VT)
-        return false;
+      if (DstRC) {
+        if (DstRC != RC)
+          return false;
+      } else
+        DstRC = RC;
       Operands.push_back("r");
     }
     return true;
   }
 
-  void PrintParameters(std::ostream &OS) const {
+  void PrintParameters(raw_ostream &OS) const {
     for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
       if (Operands[i] == "r") {
         OS << "unsigned Op" << i;
+      } else if (Operands[i] == "i") {
+        OS << "uint64_t imm" << i;
+      } else if (Operands[i] == "f") {
+        OS << "ConstantFP *f" << i;
       } else {
         assert("Unknown operand kind!");
         abort();
@@ -107,10 +135,41 @@ struct OperandsSignature {
     }
   }
 
-  void PrintArguments(std::ostream &OS) const {
+  void PrintArguments(raw_ostream &OS,
+                      const std::vector<std::string>& PR) const {
+    assert(PR.size() == Operands.size());
+    bool PrintedArg = false;
     for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
+      if (PR[i] != "")
+        // Implicit physical register operand.
+        continue;
+
+      if (PrintedArg)
+        OS << ", ";
       if (Operands[i] == "r") {
         OS << "Op" << i;
+        PrintedArg = true;
+      } else if (Operands[i] == "i") {
+        OS << "imm" << i;
+        PrintedArg = true;
+      } else if (Operands[i] == "f") {
+        OS << "f" << i;
+        PrintedArg = true;
+      } else {
+        assert("Unknown operand kind!");
+        abort();
+      }
+    }
+  }
+
+  void PrintArguments(raw_ostream &OS) const {
+    for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
+      if (Operands[i] == "r") {
+        OS << "Op" << i;
+      } else if (Operands[i] == "i") {
+        OS << "imm" << i;
+      } else if (Operands[i] == "f") {
+        OS << "f" << i;
       } else {
         assert("Unknown operand kind!");
         abort();
@@ -120,19 +179,44 @@ struct OperandsSignature {
     }
   }
 
-  void PrintManglingSuffix(std::ostream &OS) const {
+
+  void PrintManglingSuffix(raw_ostream &OS,
+                           const std::vector<std::string>& PR) const {
+    for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
+      if (PR[i] != "")
+        // Implicit physical register operand. e.g. Instruction::Mul expect to
+        // select to a binary op. On x86, mul may take a single operand with
+        // the other operand being implicit. We must emit something that looks
+        // like a binary instruction except for the very inner FastEmitInst_*
+        // call.
+        continue;
+      OS << Operands[i];
+    }
+  }
+
+  void PrintManglingSuffix(raw_ostream &OS) const {
     for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
       OS << Operands[i];
     }
   }
 };
 
-/// InstructionMemo - This class holds additional information about an
-/// instruction needed to emit code for it.
-///
-struct InstructionMemo {
-  std::string Name;
-  const CodeGenRegisterClass *RC;
+class FastISelMap {
+  typedef std::map<std::string, InstructionMemo> PredMap;
+  typedef std::map<MVT::SimpleValueType, PredMap> RetPredMap;
+  typedef std::map<MVT::SimpleValueType, RetPredMap> TypeRetPredMap;
+  typedef std::map<std::string, TypeRetPredMap> OpcodeTypeRetPredMap;
+  typedef std::map<OperandsSignature, OpcodeTypeRetPredMap> OperandsOpcodeTypeRetPredMap;
+
+  OperandsOpcodeTypeRetPredMap SimplePatterns;
+
+  std::string InstNS;
+
+public:
+  explicit FastISelMap(std::string InstNS);
+
+  void CollectPatterns(CodeGenDAGPatterns &CGP);
+  void PrintFunctionDefinitions(raw_ostream &OS);
 };
 
 }
@@ -148,39 +232,18 @@ static std::string getLegalCName(std::string OpName) {
   return OpName;
 }
 
-void FastISelEmitter::run(std::ostream &OS) {
-  EmitSourceFileHeader("\"Fast\" Instruction Selector for the " +
-                       CGP.getTargetInfo().getName() + " target", OS);
-  
-  const CodeGenTarget &Target = CGP.getTargetInfo();
-  
-  // Get the namespace to insert instructions into.  Make sure not to pick up
-  // "TargetInstrInfo" by accidentally getting the namespace off the PHI
-  // instruction or something.
-  std::string InstNS = Target.getInstNamespace();
-
-  OS << "namespace llvm {\n";
-  OS << "namespace " << InstNS << " {\n";
-  OS << "class FastISel;\n";
-  OS << "}\n";
-  OS << "}\n";
-  OS << "\n";
-  
-  if (!InstNS.empty()) InstNS += "::";
+FastISelMap::FastISelMap(std::string instns)
+  : InstNS(instns) {
+}
 
-  typedef std::map<MVT::SimpleValueType, InstructionMemo> TypeMap;
-  typedef std::map<std::string, TypeMap> OpcodeTypeMap;
-  typedef std::map<OperandsSignature, OpcodeTypeMap> OperandsOpcodeTypeMap;
-  OperandsOpcodeTypeMap SimplePatterns;
+void FastISelMap::CollectPatterns(CodeGenDAGPatterns &CGP) {
+  const CodeGenTarget &Target = CGP.getTargetInfo();
 
-  // Create the supported type signatures.
-  OperandsSignature KnownOperands;
-  SimplePatterns[KnownOperands] = OpcodeTypeMap();
-  KnownOperands.Operands.push_back("r");
-  SimplePatterns[KnownOperands] = OpcodeTypeMap();
-  KnownOperands.Operands.push_back("r");
-  SimplePatterns[KnownOperands] = OpcodeTypeMap();
+  // Determine the target's namespace name.
+  InstNS = Target.getInstNamespace() + "::";
+  assert(InstNS.size() > 2 && "Can't determine target-specific namespace!");
 
+  // Scan through all the patterns and record the simple ones.
   for (CodeGenDAGPatterns::ptm_iterator I = CGP.ptm_begin(),
        E = CGP.ptm_end(); I != E; ++I) {
     const PatternToMatch &Pattern = *I;
@@ -196,14 +259,35 @@ void FastISelEmitter::run(std::ostream &OS) {
     if (II.OperandList.empty())
       continue;
 
+    // For now, ignore multi-instruction patterns.
+    bool MultiInsts = false;
+    for (unsigned i = 0, e = Dst->getNumChildren(); i != e; ++i) {
+      TreePatternNode *ChildOp = Dst->getChild(i);
+      if (ChildOp->isLeaf())
+        continue;
+      if (ChildOp->getOperator()->isSubClassOf("Instruction")) {
+        MultiInsts = true;
+        break;
+      }
+    }
+    if (MultiInsts)
+      continue;
+
     // For now, ignore instructions where the first operand is not an
     // output register.
-    Record *Op0Rec = II.OperandList[0].Rec;
-    if (!Op0Rec->isSubClassOf("RegisterClass"))
-      continue;
-    const CodeGenRegisterClass *DstRC = &Target.getRegisterClass(Op0Rec);
-    if (!DstRC)
-      continue;
+    const CodeGenRegisterClass *DstRC = 0;
+    unsigned SubRegNo = ~0;
+    if (Op->getName() != "EXTRACT_SUBREG") {
+      Record *Op0Rec = II.OperandList[0].Rec;
+      if (!Op0Rec->isSubClassOf("RegisterClass"))
+        continue;
+      DstRC = &Target.getRegisterClass(Op0Rec);
+      if (!DstRC)
+        continue;
+    } else {
+      SubRegNo = static_cast<IntInit*>(
+                 Dst->getChild(1)->getLeafValue())->getValue();
+    }
 
     // Inspect the pattern.
     TreePatternNode *InstPatNode = Pattern.getSrcPattern();
@@ -212,145 +296,280 @@ void FastISelEmitter::run(std::ostream &OS) {
 
     Record *InstPatOp = InstPatNode->getOperator();
     std::string OpcodeName = getOpcodeName(InstPatOp, CGP);
-    MVT::SimpleValueType VT = InstPatNode->getTypeNum(0);
+    MVT::SimpleValueType RetVT = InstPatNode->getTypeNum(0);
+    MVT::SimpleValueType VT = RetVT;
+    if (InstPatNode->getNumChildren())
+      VT = InstPatNode->getChild(0)->getTypeNum(0);
 
     // For now, filter out instructions which just set a register to
     // an Operand or an immediate, like MOV32ri.
     if (InstPatOp->isSubClassOf("Operand"))
       continue;
-    if (InstPatOp->getName() == "imm" ||
-        InstPatOp->getName() == "fpimm")
-      continue;
 
     // For now, filter out any instructions with predicates.
-    if (!InstPatNode->getPredicateFn().empty())
+    if (!InstPatNode->getPredicateFns().empty())
       continue;
 
     // Check all the operands.
     OperandsSignature Operands;
-    if (!Operands.initialize(InstPatNode, Target, VT, DstRC))
-      continue;
-
-    // If it's not a known signature, ignore it.
-    if (!SimplePatterns.count(Operands))
+    if (!Operands.initialize(InstPatNode, Target, VT))
       continue;
-
-    // Ok, we found a pattern that we can handle. Remember it.
-    {
-      InstructionMemo Memo = {
-        Pattern.getDstPattern()->getOperator()->getName(),
-        DstRC
-      };
-      SimplePatterns[Operands][OpcodeName][VT] = Memo;
-    }
-  }
-
-  OS << "#include \"llvm/CodeGen/FastISel.h\"\n";
-  OS << "\n";
-  OS << "namespace llvm {\n";
-  OS << "\n";
-
-  // Declare the target FastISel class.
-  OS << "class " << InstNS << "FastISel : public llvm::FastISel {\n";
-  for (OperandsOpcodeTypeMap::const_iterator OI = SimplePatterns.begin(),
-       OE = SimplePatterns.end(); OI != OE; ++OI) {
-    const OperandsSignature &Operands = OI->first;
-    const OpcodeTypeMap &OTM = OI->second;
-
-    for (OpcodeTypeMap::const_iterator I = OTM.begin(), E = OTM.end();
-         I != E; ++I) {
-      const std::string &Opcode = I->first;
-      const TypeMap &TM = I->second;
-
-      for (TypeMap::const_iterator TI = TM.begin(), TE = TM.end();
-           TI != TE; ++TI) {
-        MVT::SimpleValueType VT = TI->first;
-
-        OS << "  unsigned FastEmit_" << getLegalCName(Opcode)
-           << "_" << getLegalCName(getName(VT)) << "(";
-        Operands.PrintParameters(OS);
-        OS << ");\n";
+    
+    std::vector<std::string>* PhysRegInputs = new std::vector<std::string>();
+    if (!InstPatNode->isLeaf() &&
+        (InstPatNode->getOperator()->getName() == "imm" ||
+         InstPatNode->getOperator()->getName() == "fpimmm"))
+      PhysRegInputs->push_back("");
+    else if (!InstPatNode->isLeaf()) {
+      for (unsigned i = 0, e = InstPatNode->getNumChildren(); i != e; ++i) {
+        TreePatternNode *Op = InstPatNode->getChild(i);
+        if (!Op->isLeaf()) {
+          PhysRegInputs->push_back("");
+          continue;
+        }
+        
+        DefInit *OpDI = dynamic_cast<DefInit*>(Op->getLeafValue());
+        Record *OpLeafRec = OpDI->getDef();
+        std::string PhysReg;
+        if (OpLeafRec->isSubClassOf("Register")) {
+          PhysReg += static_cast<StringInit*>(OpLeafRec->getValue( \
+                     "Namespace")->getValue())->getValue();
+          PhysReg += "::";
+          
+          std::vector<CodeGenRegister> Regs = Target.getRegisters();
+          for (unsigned i = 0; i < Regs.size(); ++i) {
+            if (Regs[i].TheDef == OpLeafRec) {
+              PhysReg += Regs[i].getName();
+              break;
+            }
+          }
+        }
+      
+        PhysRegInputs->push_back(PhysReg);
       }
+    } else
+      PhysRegInputs->push_back("");
 
-      OS << "  unsigned FastEmit_" << getLegalCName(Opcode)
-         << "(MVT::SimpleValueType VT";
-      if (!Operands.empty())
-        OS << ", ";
-      Operands.PrintParameters(OS);
-      OS << ");\n";
-    }
+    // Get the predicate that guards this pattern.
+    std::string PredicateCheck = Pattern.getPredicateCheck();
 
-    OS << "  unsigned FastEmit_";
-    Operands.PrintManglingSuffix(OS);
-    OS << "(MVT::SimpleValueType VT, ISD::NodeType Opcode";
-    if (!Operands.empty())
-      OS << ", ";
-    Operands.PrintParameters(OS);
-    OS << ");\n";
+    // Ok, we found a pattern that we can handle. Remember it.
+    InstructionMemo Memo = {
+      Pattern.getDstPattern()->getOperator()->getName(),
+      DstRC,
+      SubRegNo,
+      PhysRegInputs
+    };
+    assert(!SimplePatterns[Operands][OpcodeName][VT][RetVT].count(PredicateCheck) &&
+           "Duplicate pattern!");
+    SimplePatterns[Operands][OpcodeName][VT][RetVT][PredicateCheck] = Memo;
   }
-  OS << "public:\n";
-  OS << "  explicit FastISel(MachineFunction &mf) : llvm::FastISel(mf) {}\n";
-  OS << "};\n";
-  OS << "\n";
-
-  // Define the target FastISel creation function.
-  OS << "llvm::FastISel *" << InstNS
-     << "createFastISel(MachineFunction &mf) {\n";
-  OS << "  return new " << InstNS << "FastISel(mf);\n";
-  OS << "}\n";
-  OS << "\n";
+}
 
+void FastISelMap::PrintFunctionDefinitions(raw_ostream &OS) {
   // Now emit code for all the patterns that we collected.
-  for (OperandsOpcodeTypeMap::const_iterator OI = SimplePatterns.begin(),
+  for (OperandsOpcodeTypeRetPredMap::const_iterator OI = SimplePatterns.begin(),
        OE = SimplePatterns.end(); OI != OE; ++OI) {
     const OperandsSignature &Operands = OI->first;
-    const OpcodeTypeMap &OTM = OI->second;
+    const OpcodeTypeRetPredMap &OTM = OI->second;
 
-    for (OpcodeTypeMap::const_iterator I = OTM.begin(), E = OTM.end();
+    for (OpcodeTypeRetPredMap::const_iterator I = OTM.begin(), E = OTM.end();
          I != E; ++I) {
       const std::string &Opcode = I->first;
-      const TypeMap &TM = I->second;
+      const TypeRetPredMap &TM = I->second;
 
       OS << "// FastEmit functions for " << Opcode << ".\n";
       OS << "\n";
 
       // Emit one function for each opcode,type pair.
-      for (TypeMap::const_iterator TI = TM.begin(), TE = TM.end();
+      for (TypeRetPredMap::const_iterator TI = TM.begin(), TE = TM.end();
            TI != TE; ++TI) {
         MVT::SimpleValueType VT = TI->first;
-        const InstructionMemo &Memo = TI->second;
+        const RetPredMap &RM = TI->second;
+        if (RM.size() != 1) {
+          for (RetPredMap::const_iterator RI = RM.begin(), RE = RM.end();
+               RI != RE; ++RI) {
+            MVT::SimpleValueType RetVT = RI->first;
+            const PredMap &PM = RI->second;
+            bool HasPred = false;
+
+            OS << "unsigned FastEmit_"
+               << getLegalCName(Opcode)
+               << "_" << getLegalCName(getName(VT))
+               << "_" << getLegalCName(getName(RetVT)) << "_";
+            Operands.PrintManglingSuffix(OS);
+            OS << "(";
+            Operands.PrintParameters(OS);
+            OS << ") {\n";
+
+            // Emit code for each possible instruction. There may be
+            // multiple if there are subtarget concerns.
+            for (PredMap::const_iterator PI = PM.begin(), PE = PM.end();
+                 PI != PE; ++PI) {
+              std::string PredicateCheck = PI->first;
+              const InstructionMemo &Memo = PI->second;
   
-        OS << "unsigned " << InstNS << "FastISel::FastEmit_"
-           << getLegalCName(Opcode)
-           << "_" << getLegalCName(getName(VT)) << "(";
-        Operands.PrintParameters(OS);
-        OS << ") {\n";
-        OS << "  return FastEmitInst_";
-        Operands.PrintManglingSuffix(OS);
-        OS << "(" << InstNS << Memo.Name << ", ";
-        OS << InstNS << Memo.RC->getName() << "RegisterClass";
-        if (!Operands.empty())
-          OS << ", ";
-        Operands.PrintArguments(OS);
-        OS << ");\n";
-        OS << "}\n";
-        OS << "\n";
+              if (PredicateCheck.empty()) {
+                assert(!HasPred &&
+                       "Multiple instructions match, at least one has "
+                       "a predicate and at least one doesn't!");
+              } else {
+                OS << "  if (" + PredicateCheck + ") {\n";
+                OS << "  ";
+                HasPred = true;
+              }
+              
+              for (unsigned i = 0; i < Memo.PhysRegs->size(); ++i) {
+                if ((*Memo.PhysRegs)[i] != "")
+                  OS << "  TII.copyRegToReg(*MBB, MBB->end(), "
+                     << (*Memo.PhysRegs)[i] << ", Op" << i << ", "
+                     << "TM.getRegisterInfo()->getPhysicalRegisterRegClass("
+                     << (*Memo.PhysRegs)[i] << "), "
+                     << "MRI.getRegClass(Op" << i << "));\n";
+              }
+              
+              OS << "  return FastEmitInst_";
+              if (Memo.SubRegNo == (unsigned char)~0) {
+                Operands.PrintManglingSuffix(OS, *Memo.PhysRegs);
+                OS << "(" << InstNS << Memo.Name << ", ";
+                OS << InstNS << Memo.RC->getName() << "RegisterClass";
+                if (!Operands.empty())
+                  OS << ", ";
+                Operands.PrintArguments(OS, *Memo.PhysRegs);
+                OS << ");\n";
+              } else {
+                OS << "extractsubreg(" << getName(RetVT);
+                OS << ", Op0, ";
+                OS << (unsigned)Memo.SubRegNo;
+                OS << ");\n";
+              }
+              
+              if (HasPred)
+                OS << "  }\n";
+              
+            }
+            // Return 0 if none of the predicates were satisfied.
+            if (HasPred)
+              OS << "  return 0;\n";
+            OS << "}\n";
+            OS << "\n";
+          }
+          
+          // Emit one function for the type that demultiplexes on return type.
+          OS << "unsigned FastEmit_"
+             << getLegalCName(Opcode) << "_"
+             << getLegalCName(getName(VT)) << "_";
+          Operands.PrintManglingSuffix(OS);
+          OS << "(MVT RetVT";
+          if (!Operands.empty())
+            OS << ", ";
+          Operands.PrintParameters(OS);
+          OS << ") {\nswitch (RetVT.SimpleTy) {\n";
+          for (RetPredMap::const_iterator RI = RM.begin(), RE = RM.end();
+               RI != RE; ++RI) {
+            MVT::SimpleValueType RetVT = RI->first;
+            OS << "  case " << getName(RetVT) << ": return FastEmit_"
+               << getLegalCName(Opcode) << "_" << getLegalCName(getName(VT))
+               << "_" << getLegalCName(getName(RetVT)) << "_";
+            Operands.PrintManglingSuffix(OS);
+            OS << "(";
+            Operands.PrintArguments(OS);
+            OS << ");\n";
+          }
+          OS << "  default: return 0;\n}\n}\n\n";
+          
+        } else {
+          // Non-variadic return type.
+          OS << "unsigned FastEmit_"
+             << getLegalCName(Opcode) << "_"
+             << getLegalCName(getName(VT)) << "_";
+          Operands.PrintManglingSuffix(OS);
+          OS << "(MVT RetVT";
+          if (!Operands.empty())
+            OS << ", ";
+          Operands.PrintParameters(OS);
+          OS << ") {\n";
+          
+          OS << "  if (RetVT.SimpleTy != " << getName(RM.begin()->first)
+             << ")\n    return 0;\n";
+          
+          const PredMap &PM = RM.begin()->second;
+          bool HasPred = false;
+          
+          // Emit code for each possible instruction. There may be
+          // multiple if there are subtarget concerns.
+          for (PredMap::const_iterator PI = PM.begin(), PE = PM.end(); PI != PE;
+               ++PI) {
+            std::string PredicateCheck = PI->first;
+            const InstructionMemo &Memo = PI->second;
+
+            if (PredicateCheck.empty()) {
+              assert(!HasPred &&
+                     "Multiple instructions match, at least one has "
+                     "a predicate and at least one doesn't!");
+            } else {
+              OS << "  if (" + PredicateCheck + ") {\n";
+              OS << "  ";
+              HasPred = true;
+            }
+            
+             for (unsigned i = 0; i < Memo.PhysRegs->size(); ++i) {
+                if ((*Memo.PhysRegs)[i] != "")
+                  OS << "  TII.copyRegToReg(*MBB, MBB->end(), "
+                     << (*Memo.PhysRegs)[i] << ", Op" << i << ", "
+                     << "TM.getRegisterInfo()->getPhysicalRegisterRegClass("
+                     << (*Memo.PhysRegs)[i] << "), "
+                     << "MRI.getRegClass(Op" << i << "));\n";
+              }
+            
+            OS << "  return FastEmitInst_";
+            
+            if (Memo.SubRegNo == (unsigned char)~0) {
+              Operands.PrintManglingSuffix(OS, *Memo.PhysRegs);
+              OS << "(" << InstNS << Memo.Name << ", ";
+              OS << InstNS << Memo.RC->getName() << "RegisterClass";
+              if (!Operands.empty())
+                OS << ", ";
+              Operands.PrintArguments(OS, *Memo.PhysRegs);
+              OS << ");\n";
+            } else {
+              OS << "extractsubreg(RetVT, Op0, ";
+              OS << (unsigned)Memo.SubRegNo;
+              OS << ");\n";
+            }
+            
+             if (HasPred)
+               OS << "  }\n";
+          }
+          
+          // Return 0 if none of the predicates were satisfied.
+          if (HasPred)
+            OS << "  return 0;\n";
+          OS << "}\n";
+          OS << "\n";
+        }
       }
 
       // Emit one function for the opcode that demultiplexes based on the type.
-      OS << "unsigned " << InstNS << "FastISel::FastEmit_"
-         << getLegalCName(Opcode) << "(MVT::SimpleValueType VT";
+      OS << "unsigned FastEmit_"
+         << getLegalCName(Opcode) << "_";
+      Operands.PrintManglingSuffix(OS);
+      OS << "(MVT VT, MVT RetVT";
       if (!Operands.empty())
         OS << ", ";
       Operands.PrintParameters(OS);
       OS << ") {\n";
-      OS << "  switch (VT) {\n";
-      for (TypeMap::const_iterator TI = TM.begin(), TE = TM.end();
+      OS << "  switch (VT.SimpleTy) {\n";
+      for (TypeRetPredMap::const_iterator TI = TM.begin(), TE = TM.end();
            TI != TE; ++TI) {
         MVT::SimpleValueType VT = TI->first;
         std::string TypeName = getName(VT);
         OS << "  case " << TypeName << ": return FastEmit_"
-           << getLegalCName(Opcode) << "_" << getLegalCName(TypeName) << "(";
+           << getLegalCName(Opcode) << "_" << getLegalCName(TypeName) << "_";
+        Operands.PrintManglingSuffix(OS);
+        OS << "(RetVT";
+        if (!Operands.empty())
+          OS << ", ";
         Operands.PrintArguments(OS);
         OS << ");\n";
       }
@@ -360,22 +579,27 @@ void FastISelEmitter::run(std::ostream &OS) {
       OS << "\n";
     }
 
+    OS << "// Top-level FastEmit function.\n";
+    OS << "\n";
+
     // Emit one function for the operand signature that demultiplexes based
     // on opcode and type.
-    OS << "unsigned " << InstNS << "FastISel::FastEmit_";
+    OS << "unsigned FastEmit_";
     Operands.PrintManglingSuffix(OS);
-    OS << "(MVT::SimpleValueType VT, ISD::NodeType Opcode";
+    OS << "(MVT VT, MVT RetVT, unsigned Opcode";
     if (!Operands.empty())
       OS << ", ";
     Operands.PrintParameters(OS);
     OS << ") {\n";
     OS << "  switch (Opcode) {\n";
-    for (OpcodeTypeMap::const_iterator I = OTM.begin(), E = OTM.end();
+    for (OpcodeTypeRetPredMap::const_iterator I = OTM.begin(), E = OTM.end();
          I != E; ++I) {
       const std::string &Opcode = I->first;
 
       OS << "  case " << Opcode << ": return FastEmit_"
-         << getLegalCName(Opcode) << "(VT";
+         << getLegalCName(Opcode) << "_";
+      Operands.PrintManglingSuffix(OS);
+      OS << "(VT, RetVT";
       if (!Operands.empty())
         OS << ", ";
       Operands.PrintArguments(OS);
@@ -386,8 +610,25 @@ void FastISelEmitter::run(std::ostream &OS) {
     OS << "}\n";
     OS << "\n";
   }
+}
+
+void FastISelEmitter::run(raw_ostream &OS) {
+  const CodeGenTarget &Target = CGP.getTargetInfo();
+
+  // Determine the target's namespace name.
+  std::string InstNS = Target.getInstNamespace() + "::";
+  assert(InstNS.size() > 2 && "Can't determine target-specific namespace!");
+
+  EmitSourceFileHeader("\"Fast\" Instruction Selector for the " +
+                       Target.getName() + " target", OS);
+
+  FastISelMap F(InstNS);
+  F.CollectPatterns(CGP);
+  F.PrintFunctionDefinitions(OS);
+}
 
-  OS << "}\n";
+FastISelEmitter::FastISelEmitter(RecordKeeper &R)
+  : Records(R),
+    CGP(R) {
 }
 
-// todo: really filter out Constants