X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAGISel.cpp;h=68b9146adfe1ad96d41f8d34ef4905dd41ae4957;hb=6079f00035e029f12e4be0281aa2cbfbb4817141;hp=1731be2be9581494b2a33c6bcb6ae2d2e18add3e;hpb=70017e44cdba1946cc478ce1856a3e855a767e28;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 1731be2be95..faca3ee823c 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -12,23 +12,19 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "isel" +#include "llvm/CodeGen/SelectionDAGISel.h" #include "ScheduleDAGSDNodes.h" #include "SelectionDAGBuilder.h" -#include "llvm/CodeGen/FunctionLoweringInfo.h" -#include "llvm/CodeGen/SelectionDAGISel.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/DebugInfo.h" -#include "llvm/Constants.h" -#include "llvm/Function.h" -#include "llvm/InlineAsm.h" -#include "llvm/Instructions.h" -#include "llvm/Intrinsics.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" -#include "llvm/Module.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/CFG.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/FastISel.h" -#include "llvm/CodeGen/GCStrategy.h" +#include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/GCMetadata.h" +#include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -37,23 +33,115 @@ #include "llvm/CodeGen/ScheduleHazardRecognizer.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetIntrinsicInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetIntrinsicInfo.h" +#include "llvm/Target/TargetLibraryInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include using namespace llvm; STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on"); +STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected"); +STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel"); +STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG"); STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path"); +STATISTIC(NumEntryBlocks, "Number of entry blocks encountered"); +STATISTIC(NumFastIselFailLowerArguments, + "Number of entry blocks where fast isel failed to lower arguments"); + +#ifndef NDEBUG +static cl::opt +EnableFastISelVerbose2("fast-isel-verbose2", cl::Hidden, + cl::desc("Enable extra verbose messages in the \"fast\" " + "instruction selector")); + + // Terminators +STATISTIC(NumFastIselFailRet,"Fast isel fails on Ret"); +STATISTIC(NumFastIselFailBr,"Fast isel fails on Br"); +STATISTIC(NumFastIselFailSwitch,"Fast isel fails on Switch"); +STATISTIC(NumFastIselFailIndirectBr,"Fast isel fails on IndirectBr"); +STATISTIC(NumFastIselFailInvoke,"Fast isel fails on Invoke"); +STATISTIC(NumFastIselFailResume,"Fast isel fails on Resume"); +STATISTIC(NumFastIselFailUnreachable,"Fast isel fails on Unreachable"); + + // Standard binary operators... +STATISTIC(NumFastIselFailAdd,"Fast isel fails on Add"); +STATISTIC(NumFastIselFailFAdd,"Fast isel fails on FAdd"); +STATISTIC(NumFastIselFailSub,"Fast isel fails on Sub"); +STATISTIC(NumFastIselFailFSub,"Fast isel fails on FSub"); +STATISTIC(NumFastIselFailMul,"Fast isel fails on Mul"); +STATISTIC(NumFastIselFailFMul,"Fast isel fails on FMul"); +STATISTIC(NumFastIselFailUDiv,"Fast isel fails on UDiv"); +STATISTIC(NumFastIselFailSDiv,"Fast isel fails on SDiv"); +STATISTIC(NumFastIselFailFDiv,"Fast isel fails on FDiv"); +STATISTIC(NumFastIselFailURem,"Fast isel fails on URem"); +STATISTIC(NumFastIselFailSRem,"Fast isel fails on SRem"); +STATISTIC(NumFastIselFailFRem,"Fast isel fails on FRem"); + + // Logical operators... +STATISTIC(NumFastIselFailAnd,"Fast isel fails on And"); +STATISTIC(NumFastIselFailOr,"Fast isel fails on Or"); +STATISTIC(NumFastIselFailXor,"Fast isel fails on Xor"); + + // Memory instructions... +STATISTIC(NumFastIselFailAlloca,"Fast isel fails on Alloca"); +STATISTIC(NumFastIselFailLoad,"Fast isel fails on Load"); +STATISTIC(NumFastIselFailStore,"Fast isel fails on Store"); +STATISTIC(NumFastIselFailAtomicCmpXchg,"Fast isel fails on AtomicCmpXchg"); +STATISTIC(NumFastIselFailAtomicRMW,"Fast isel fails on AtomicRWM"); +STATISTIC(NumFastIselFailFence,"Fast isel fails on Frence"); +STATISTIC(NumFastIselFailGetElementPtr,"Fast isel fails on GetElementPtr"); + + // Convert instructions... +STATISTIC(NumFastIselFailTrunc,"Fast isel fails on Trunc"); +STATISTIC(NumFastIselFailZExt,"Fast isel fails on ZExt"); +STATISTIC(NumFastIselFailSExt,"Fast isel fails on SExt"); +STATISTIC(NumFastIselFailFPTrunc,"Fast isel fails on FPTrunc"); +STATISTIC(NumFastIselFailFPExt,"Fast isel fails on FPExt"); +STATISTIC(NumFastIselFailFPToUI,"Fast isel fails on FPToUI"); +STATISTIC(NumFastIselFailFPToSI,"Fast isel fails on FPToSI"); +STATISTIC(NumFastIselFailUIToFP,"Fast isel fails on UIToFP"); +STATISTIC(NumFastIselFailSIToFP,"Fast isel fails on SIToFP"); +STATISTIC(NumFastIselFailIntToPtr,"Fast isel fails on IntToPtr"); +STATISTIC(NumFastIselFailPtrToInt,"Fast isel fails on PtrToInt"); +STATISTIC(NumFastIselFailBitCast,"Fast isel fails on BitCast"); + + // Other instructions... +STATISTIC(NumFastIselFailICmp,"Fast isel fails on ICmp"); +STATISTIC(NumFastIselFailFCmp,"Fast isel fails on FCmp"); +STATISTIC(NumFastIselFailPHI,"Fast isel fails on PHI"); +STATISTIC(NumFastIselFailSelect,"Fast isel fails on Select"); +STATISTIC(NumFastIselFailCall,"Fast isel fails on Call"); +STATISTIC(NumFastIselFailShl,"Fast isel fails on Shl"); +STATISTIC(NumFastIselFailLShr,"Fast isel fails on LShr"); +STATISTIC(NumFastIselFailAShr,"Fast isel fails on AShr"); +STATISTIC(NumFastIselFailVAArg,"Fast isel fails on VAArg"); +STATISTIC(NumFastIselFailExtractElement,"Fast isel fails on ExtractElement"); +STATISTIC(NumFastIselFailInsertElement,"Fast isel fails on InsertElement"); +STATISTIC(NumFastIselFailShuffleVector,"Fast isel fails on ShuffleVector"); +STATISTIC(NumFastIselFailExtractValue,"Fast isel fails on ExtractValue"); +STATISTIC(NumFastIselFailInsertValue,"Fast isel fails on InsertValue"); +STATISTIC(NumFastIselFailLandingPad,"Fast isel fails on LandingPad"); +#endif static cl::opt EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, @@ -61,7 +149,17 @@ EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, "instruction selector")); static cl::opt EnableFastISelAbort("fast-isel-abort", cl::Hidden, - cl::desc("Enable abort calls when \"fast\" instruction fails")); + cl::desc("Enable abort calls when \"fast\" instruction selection " + "fails to lower an instruction")); +static cl::opt +EnableFastISelAbortArgs("fast-isel-abort-args", cl::Hidden, + cl::desc("Enable abort calls when \"fast\" instruction selection " + "fails to lower a formal argument")); + +static cl::opt +UseMBPI("use-mbpi", + cl::desc("use Machine Branch Probability Info"), + cl::init(true), cl::Hidden); #ifndef NDEBUG static cl::opt @@ -124,22 +222,57 @@ defaultListDAGScheduler("default", "Best scheduler for the target", createDefaultScheduler); namespace llvm { + //===--------------------------------------------------------------------===// + /// \brief This struct is used by SelectionDAGISel to temporarily override + /// the optimization level on a per-function basis. + class OptLevelChanger { + SelectionDAGISel &IS; + CodeGenOpt::Level SavedOptLevel; + + public: + OptLevelChanger(SelectionDAGISel &ISel, + CodeGenOpt::Level NewOptLevel) : IS(ISel) { + SavedOptLevel = IS.OptLevel; + if (NewOptLevel == SavedOptLevel) + return; + IS.OptLevel = NewOptLevel; + IS.TM.setOptLevel(NewOptLevel); + DEBUG(dbgs() << "\nChanging optimization level for Function " + << IS.MF->getFunction()->getName() << "\n"); + DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel + << " ; After: -O" << NewOptLevel << "\n"); + } + + ~OptLevelChanger() { + if (IS.OptLevel == SavedOptLevel) + return; + DEBUG(dbgs() << "\nRestoring optimization level for Function " + << IS.MF->getFunction()->getName() << "\n"); + DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel + << " ; After: -O" << SavedOptLevel << "\n"); + IS.OptLevel = SavedOptLevel; + IS.TM.setOptLevel(SavedOptLevel); + } + }; + //===--------------------------------------------------------------------===// /// createDefaultScheduler - This creates an instruction scheduler appropriate /// for the target. ScheduleDAGSDNodes* createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel) { - const TargetLowering &TLI = IS->getTargetLowering(); + const TargetLowering *TLI = IS->getTargetLowering(); + const TargetSubtargetInfo &ST = IS->TM.getSubtarget(); - if (OptLevel == CodeGenOpt::None) + if (OptLevel == CodeGenOpt::None || ST.useMachineScheduler() || + TLI->getSchedulingPreference() == Sched::Source) return createSourceListDAGScheduler(IS, OptLevel); - if (TLI.getSchedulingPreference() == Sched::Latency) - return createTDListDAGScheduler(IS, OptLevel); - if (TLI.getSchedulingPreference() == Sched::RegPressure) + if (TLI->getSchedulingPreference() == Sched::RegPressure) return createBURRListDAGScheduler(IS, OptLevel); - if (TLI.getSchedulingPreference() == Sched::Hybrid) + if (TLI->getSchedulingPreference() == Sched::Hybrid) return createHybridListDAGScheduler(IS, OptLevel); - assert(TLI.getSchedulingPreference() == Sched::ILP && + if (TLI->getSchedulingPreference() == Sched::VLIW) + return createVLIWDAGScheduler(IS, OptLevel); + assert(TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!"); return createILPListDAGScheduler(IS, OptLevel); } @@ -163,22 +296,33 @@ TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, "TargetLowering::EmitInstrWithCustomInserter!"; #endif llvm_unreachable(0); - return 0; +} + +void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr *MI, + SDNode *Node) const { + assert(!MI->hasPostISelHook() && + "If a target marks an instruction with 'hasPostISelHook', " + "it must implement TargetLowering::AdjustInstrPostInstrSelection!"); } //===----------------------------------------------------------------------===// // SelectionDAGISel code //===----------------------------------------------------------------------===// -SelectionDAGISel::SelectionDAGISel(const TargetMachine &tm, CodeGenOpt::Level OL) : - MachineFunctionPass(&ID), TM(tm), TLI(*tm.getTargetLowering()), - FuncInfo(new FunctionLoweringInfo(TLI)), - CurDAG(new SelectionDAG(tm)), +SelectionDAGISel::SelectionDAGISel(TargetMachine &tm, + CodeGenOpt::Level OL) : + MachineFunctionPass(ID), TM(tm), + FuncInfo(new FunctionLoweringInfo(TM)), + CurDAG(new SelectionDAG(tm, OL)), SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)), GFI(), OptLevel(OL), - DAGSize(0) -{} + DAGSize(0) { + initializeGCModuleInfoPass(*PassRegistry::getPassRegistry()); + initializeAliasAnalysisAnalysisGroup(*PassRegistry::getPassRegistry()); + initializeBranchProbabilityInfoPass(*PassRegistry::getPassRegistry()); + initializeTargetLibraryInfoPass(*PassRegistry::getPassRegistry()); + } SelectionDAGISel::~SelectionDAGISel() { delete SDB; @@ -191,64 +335,95 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { AU.addPreserved(); AU.addRequired(); AU.addPreserved(); + AU.addRequired(); + if (UseMBPI && OptLevel != CodeGenOpt::None) + AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); } -/// FunctionCallsSetJmp - Return true if the function has a call to setjmp or -/// other function that gcc recognizes as "returning twice". This is used to -/// limit code-gen optimizations on the machine function. +/// SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that +/// may trap on it. In this case we have to split the edge so that the path +/// through the predecessor block that doesn't go to the phi block doesn't +/// execute the possibly trapping instruction. /// -/// FIXME: Remove after is fixed. -static bool FunctionCallsSetJmp(const Function *F) { - const Module *M = F->getParent(); - static const char *ReturnsTwiceFns[] = { - "setjmp", - "sigsetjmp", - "setjmp_syscall", - "savectx", - "qsetjmp", - "vfork", - "getcontext" - }; -#define NUM_RETURNS_TWICE_FNS sizeof(ReturnsTwiceFns) / sizeof(const char *) - - for (unsigned I = 0; I < NUM_RETURNS_TWICE_FNS; ++I) - if (const Function *Callee = M->getFunction(ReturnsTwiceFns[I])) { - if (!Callee->use_empty()) - for (Value::const_use_iterator - I = Callee->use_begin(), E = Callee->use_end(); - I != E; ++I) - if (const CallInst *CI = dyn_cast(*I)) - if (CI->getParent()->getParent() == F) - return true; - } +/// This is required for correctness, so it must be done at -O0. +/// +static void SplitCriticalSideEffectEdges(Function &Fn, Pass *SDISel) { + // Loop for blocks with phi nodes. + for (Function::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { + PHINode *PN = dyn_cast(BB->begin()); + if (PN == 0) continue; + + ReprocessBlock: + // For each block with a PHI node, check to see if any of the input values + // are potentially trapping constant expressions. Constant expressions are + // the only potentially trapping value that can occur as the argument to a + // PHI. + for (BasicBlock::iterator I = BB->begin(); (PN = dyn_cast(I)); ++I) + for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { + ConstantExpr *CE = dyn_cast(PN->getIncomingValue(i)); + if (CE == 0 || !CE->canTrap()) continue; + + // The only case we have to worry about is when the edge is critical. + // Since this block has a PHI Node, we assume it has multiple input + // edges: check to see if the pred has multiple successors. + BasicBlock *Pred = PN->getIncomingBlock(i); + if (Pred->getTerminator()->getNumSuccessors() == 1) + continue; - return false; -#undef NUM_RETURNS_TWICE_FNS + // Okay, we have to split this edge. + SplitCriticalEdge(Pred->getTerminator(), + GetSuccessorNumber(Pred, BB), SDISel, true); + goto ReprocessBlock; + } + } } bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // Do some sanity-checking on the command-line options. - assert((!EnableFastISelVerbose || EnableFastISel) && + assert((!EnableFastISelVerbose || TM.Options.EnableFastISel) && "-fast-isel-verbose requires -fast-isel"); - assert((!EnableFastISelAbort || EnableFastISel) && + assert((!EnableFastISelAbort || TM.Options.EnableFastISel) && "-fast-isel-abort requires -fast-isel"); const Function &Fn = *mf.getFunction(); const TargetInstrInfo &TII = *TM.getInstrInfo(); const TargetRegisterInfo &TRI = *TM.getRegisterInfo(); + const TargetLowering *TLI = TM.getTargetLowering(); MF = &mf; RegInfo = &MF->getRegInfo(); AA = &getAnalysis(); + LibInfo = &getAnalysis(); + TTI = getAnalysisIfAvailable(); GFI = Fn.hasGC() ? &getAnalysis().getFunctionInfo(Fn) : 0; + TargetSubtargetInfo &ST = + const_cast(TM.getSubtarget()); + ST.resetSubtargetFeatures(MF); + TM.resetTargetOptions(MF); + + // Reset OptLevel to None for optnone functions. + CodeGenOpt::Level NewOptLevel = OptLevel; + if (Fn.hasFnAttribute(Attribute::OptimizeNone)) + NewOptLevel = CodeGenOpt::None; + OptLevelChanger OLC(*this, NewOptLevel); + DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n"); - CurDAG->init(*MF); + SplitCriticalSideEffectEdges(const_cast(Fn), this); + + CurDAG->init(*MF, TTI, TLI); FuncInfo->set(Fn, *MF); - SDB->init(GFI, *AA); + if (UseMBPI && OptLevel != CodeGenOpt::None) + FuncInfo->BPI = &getAnalysis(); + else + FuncInfo->BPI = 0; + + SDB->init(GFI, *AA, LibInfo); + + MF->setHasMSInlineAsm(false); SelectAllBasicBlocks(Fn); // If the first basic block in the function has live ins that need to be @@ -261,62 +436,96 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { if (!FuncInfo->ArgDbgValues.empty()) for (MachineRegisterInfo::livein_iterator LI = RegInfo->livein_begin(), E = RegInfo->livein_end(); LI != E; ++LI) - if (LI->second) + if (LI->second) LiveInMap.insert(std::make_pair(LI->first, LI->second)); // Insert DBG_VALUE instructions for function arguments to the entry block. for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) { MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1]; - unsigned Reg = MI->getOperand(0).getReg(); + bool hasFI = MI->getOperand(0).isFI(); + unsigned Reg = + hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) EntryMBB->insert(EntryMBB->begin(), MI); else { MachineInstr *Def = RegInfo->getVRegDef(Reg); - MachineBasicBlock::iterator InsertPos = Def; - // FIXME: VR def may not be in entry block. - Def->getParent()->insert(llvm::next(InsertPos), MI); + if (Def) { + MachineBasicBlock::iterator InsertPos = Def; + // FIXME: VR def may not be in entry block. + Def->getParent()->insert(llvm::next(InsertPos), MI); + } else + DEBUG(dbgs() << "Dropping debug info for dead vreg" + << TargetRegisterInfo::virtReg2Index(Reg) << "\n"); } // If Reg is live-in then update debug info to track its copy in a vreg. DenseMap::iterator LDI = LiveInMap.find(Reg); if (LDI != LiveInMap.end()) { + assert(!hasFI && "There's no handling of frame pointer updating here yet " + "- add if needed"); MachineInstr *Def = RegInfo->getVRegDef(LDI->second); MachineBasicBlock::iterator InsertPos = Def; - const MDNode *Variable = + const MDNode *Variable = MI->getOperand(MI->getNumOperands()-1).getMetadata(); - unsigned Offset = MI->getOperand(1).getImm(); + bool IsIndirect = MI->isIndirectDebugValue(); + unsigned Offset = IsIndirect ? MI->getOperand(1).getImm() : 0; // Def is never a terminator here, so it is ok to increment InsertPos. - BuildMI(*EntryMBB, ++InsertPos, MI->getDebugLoc(), - TII.get(TargetOpcode::DBG_VALUE)) - .addReg(LDI->second, RegState::Debug) - .addImm(Offset).addMetadata(Variable); + BuildMI(*EntryMBB, ++InsertPos, MI->getDebugLoc(), + TII.get(TargetOpcode::DBG_VALUE), + IsIndirect, + LDI->second, Offset, Variable); + + // If this vreg is directly copied into an exported register then + // that COPY instructions also need DBG_VALUE, if it is the only + // user of LDI->second. + MachineInstr *CopyUseMI = NULL; + for (MachineRegisterInfo::use_iterator + UI = RegInfo->use_begin(LDI->second); + MachineInstr *UseMI = UI.skipInstruction();) { + if (UseMI->isDebugValue()) continue; + if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) { + CopyUseMI = UseMI; continue; + } + // Otherwise this is another use or second copy use. + CopyUseMI = NULL; break; + } + if (CopyUseMI) { + MachineInstr *NewMI = + BuildMI(*MF, CopyUseMI->getDebugLoc(), + TII.get(TargetOpcode::DBG_VALUE), + IsIndirect, + CopyUseMI->getOperand(0).getReg(), + Offset, Variable); + MachineBasicBlock::iterator Pos = CopyUseMI; + EntryMBB->insertAfter(Pos, NewMI); + } } } // Determine if there are any calls in this machine function. MachineFrameInfo *MFI = MF->getFrameInfo(); - if (!MFI->hasCalls()) { - for (MachineFunction::const_iterator - I = MF->begin(), E = MF->end(); I != E; ++I) { - const MachineBasicBlock *MBB = I; - for (MachineBasicBlock::const_iterator - II = MBB->begin(), IE = MBB->end(); II != IE; ++II) { - const TargetInstrDesc &TID = TM.getInstrInfo()->get(II->getOpcode()); - - // Operand 1 of an inline asm instruction indicates whether the asm - // needs stack or not. - if ((II->isInlineAsm() && II->getOperand(1).getImm()) || - (TID.isCall() && !TID.isReturn())) { - MFI->setHasCalls(true); - goto done; - } + for (MachineFunction::const_iterator I = MF->begin(), E = MF->end(); I != E; + ++I) { + + if (MFI->hasCalls() && MF->hasMSInlineAsm()) + break; + + const MachineBasicBlock *MBB = I; + for (MachineBasicBlock::const_iterator II = MBB->begin(), IE = MBB->end(); + II != IE; ++II) { + const MCInstrDesc &MCID = TM.getInstrInfo()->get(II->getOpcode()); + if ((MCID.isCall() && !MCID.isReturn()) || + II->isStackAligningInlineAsm()) { + MFI->setHasCalls(true); + } + if (II->isMSInlineAsm()) { + MF->setHasMSInlineAsm(true); } } - done:; } // Determine if there is a call to setjmp in the machine function. - MF->setCallsSetJmp(FunctionCallsSetJmp(&Fn)); + MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice()); // Replace forward-declared registers with the registers containing // the desired value. @@ -329,15 +538,23 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // If To is also scheduled to be replaced, find what its ultimate // replacement is. for (;;) { - DenseMap::iterator J = - FuncInfo->RegFixups.find(To); + DenseMap::iterator J = FuncInfo->RegFixups.find(To); if (J == E) break; To = J->second; } + // Make sure the new register has a sufficiently constrained register class. + if (TargetRegisterInfo::isVirtualRegister(From) && + TargetRegisterInfo::isVirtualRegister(To)) + MRI.constrainRegClass(To, MRI.getRegClass(From)); // Replace it. MRI.replaceRegWith(From, To); } + // Freeze the set of reserved registers now that MachineFrameInfo has been + // set up. All the information required by getReservedRegs() should be + // available now. + MRI.freezeReservedRegs(*MF); + // Release function-specific state. SDB and CurDAG are already cleared // at this point. FuncInfo->clear(); @@ -345,10 +562,9 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { return true; } -void -SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin, - BasicBlock::const_iterator End, - bool &HadTailCall) { +void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin, + BasicBlock::const_iterator End, + bool &HadTailCall) { // Lower all of the non-terminator instructions. If a call is emitted // as a tail call, cease emitting nodes for this block. Terminators // are handled below. @@ -364,45 +580,12 @@ SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin, CodeGenAndEmitDAG(); } -namespace { -/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted -/// nodes from the worklist. -class SDOPsWorkListRemover : public SelectionDAG::DAGUpdateListener { - SmallVector &Worklist; - SmallPtrSet &InWorklist; -public: - SDOPsWorkListRemover(SmallVector &wl, - SmallPtrSet &inwl) - : Worklist(wl), InWorklist(inwl) {} - - void RemoveFromWorklist(SDNode *N) { - if (!InWorklist.erase(N)) return; - - SmallVector::iterator I = - std::find(Worklist.begin(), Worklist.end(), N); - assert(I != Worklist.end() && "Not in worklist"); - - *I = Worklist.back(); - Worklist.pop_back(); - } - - virtual void NodeDeleted(SDNode *N, SDNode *E) { - RemoveFromWorklist(N); - } - - virtual void NodeUpdated(SDNode *N) { - // Ignore updates. - } -}; -} - void SelectionDAGISel::ComputeLiveOutVRegInfo() { SmallPtrSet VisitedNodes; SmallVector Worklist; Worklist.push_back(CurDAG->getRoot().getNode()); - APInt Mask; APInt KnownZero; APInt KnownOne; @@ -433,20 +616,8 @@ void SelectionDAGISel::ComputeLiveOutVRegInfo() { continue; unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src); - Mask = APInt::getAllOnesValue(SrcVT.getSizeInBits()); - CurDAG->ComputeMaskedBits(Src, Mask, KnownZero, KnownOne); - - // Only install this information if it tells us something. - if (NumSignBits != 1 || KnownZero != 0 || KnownOne != 0) { - DestReg -= TargetRegisterInfo::FirstVirtualRegister; - if (DestReg >= FuncInfo->LiveOutRegInfo.size()) - FuncInfo->LiveOutRegInfo.resize(DestReg+1); - FunctionLoweringInfo::LiveOutInfo &LOI = - FuncInfo->LiveOutRegInfo[DestReg]; - LOI.NumSignBits = NumSignBits; - LOI.KnownOne = KnownOne; - LOI.KnownZero = KnownZero; - } + CurDAG->ComputeMaskedBits(Src, KnownZero, KnownOne); + FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, KnownZero, KnownOne); } while (!Worklist.empty()); } @@ -455,23 +626,31 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { if (TimePassesIsEnabled) GroupName = "Instruction Selection and Scheduling"; std::string BlockName; + int BlockNumber = -1; + (void)BlockNumber; +#ifdef NDEBUG if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs || ViewDAGCombine2 || ViewDAGCombineLT || ViewISelDAGs || ViewSchedDAGs || ViewSUnitDAGs) - BlockName = MF->getFunction()->getNameStr() + ":" + - FuncInfo->MBB->getBasicBlock()->getNameStr(); - - DEBUG(dbgs() << "Initial selection DAG:\n"; CurDAG->dump()); +#endif + { + BlockNumber = FuncInfo->MBB->getNumber(); + BlockName = MF->getName().str() + ":" + + FuncInfo->MBB->getBasicBlock()->getName().str(); + } + DEBUG(dbgs() << "Initial selection DAG: BB#" << BlockNumber + << " '" << BlockName << "'\n"; CurDAG->dump()); if (ViewDAGCombine1) CurDAG->viewGraph("dag-combine1 input for " + BlockName); // Run the DAG combiner in pre-legalize mode. { NamedRegionTimer T("DAG Combining 1", GroupName, TimePassesIsEnabled); - CurDAG->Combine(Unrestricted, *AA, OptLevel); + CurDAG->Combine(BeforeLegalizeTypes, *AA, OptLevel); } - DEBUG(dbgs() << "Optimized lowered selection DAG:\n"; CurDAG->dump()); + DEBUG(dbgs() << "Optimized lowered selection DAG: BB#" << BlockNumber + << " '" << BlockName << "'\n"; CurDAG->dump()); // Second step, hack on the DAG until it only uses operations and types that // the target supports. @@ -484,7 +663,8 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { Changed = CurDAG->LegalizeTypes(); } - DEBUG(dbgs() << "Type-legalized selection DAG:\n"; CurDAG->dump()); + DEBUG(dbgs() << "Type-legalized selection DAG: BB#" << BlockNumber + << " '" << BlockName << "'\n"; CurDAG->dump()); if (Changed) { if (ViewDAGCombineLT) @@ -494,11 +674,12 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { { NamedRegionTimer T("DAG Combining after legalize types", GroupName, TimePassesIsEnabled); - CurDAG->Combine(NoIllegalTypes, *AA, OptLevel); + CurDAG->Combine(AfterLegalizeTypes, *AA, OptLevel); } - DEBUG(dbgs() << "Optimized type-legalized selection DAG:\n"; - CurDAG->dump()); + DEBUG(dbgs() << "Optimized type-legalized selection DAG: BB#" << BlockNumber + << " '" << BlockName << "'\n"; CurDAG->dump()); + } { @@ -519,31 +700,35 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { { NamedRegionTimer T("DAG Combining after legalize vectors", GroupName, TimePassesIsEnabled); - CurDAG->Combine(NoIllegalOperations, *AA, OptLevel); + CurDAG->Combine(AfterLegalizeVectorOps, *AA, OptLevel); } - DEBUG(dbgs() << "Optimized vector-legalized selection DAG:\n"; - CurDAG->dump()); + DEBUG(dbgs() << "Optimized vector-legalized selection DAG: BB#" + << BlockNumber << " '" << BlockName << "'\n"; CurDAG->dump()); } + CurDAG->NewNodesMustHaveLegalTypes = true; + if (ViewLegalizeDAGs) CurDAG->viewGraph("legalize input for " + BlockName); { NamedRegionTimer T("DAG Legalization", GroupName, TimePassesIsEnabled); - CurDAG->Legalize(OptLevel); + CurDAG->Legalize(); } - DEBUG(dbgs() << "Legalized selection DAG:\n"; CurDAG->dump()); + DEBUG(dbgs() << "Legalized selection DAG: BB#" << BlockNumber + << " '" << BlockName << "'\n"; CurDAG->dump()); if (ViewDAGCombine2) CurDAG->viewGraph("dag-combine2 input for " + BlockName); // Run the DAG combiner in post-legalize mode. { NamedRegionTimer T("DAG Combining 2", GroupName, TimePassesIsEnabled); - CurDAG->Combine(NoIllegalOperations, *AA, OptLevel); + CurDAG->Combine(AfterLegalizeDAG, *AA, OptLevel); } - DEBUG(dbgs() << "Optimized legalized selection DAG:\n"; CurDAG->dump()); + DEBUG(dbgs() << "Optimized legalized selection DAG: BB#" << BlockNumber + << " '" << BlockName << "'\n"; CurDAG->dump()); if (OptLevel != CodeGenOpt::None) ComputeLiveOutVRegInfo(); @@ -557,7 +742,8 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { DoInstructionSelection(); } - DEBUG(dbgs() << "Selected selection DAG:\n"; CurDAG->dump()); + DEBUG(dbgs() << "Selected selection DAG: BB#" << BlockNumber + << " '" << BlockName << "'\n"; CurDAG->dump()); if (ViewSchedDAGs) CurDAG->viewGraph("scheduler input for " + BlockName); @@ -566,20 +752,27 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { { NamedRegionTimer T("Instruction Scheduling", GroupName, TimePassesIsEnabled); - Scheduler->Run(CurDAG, FuncInfo->MBB, FuncInfo->InsertPt); + Scheduler->Run(CurDAG, FuncInfo->MBB); } if (ViewSUnitDAGs) Scheduler->viewGraph(); // Emit machine code to BB. This can change 'BB' to the last block being // inserted into. + MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB; { NamedRegionTimer T("Instruction Creation", GroupName, TimePassesIsEnabled); - FuncInfo->MBB = Scheduler->EmitSchedule(); - FuncInfo->InsertPt = Scheduler->InsertPos; + // FuncInfo->InsertPt is passed by reference and set to the end of the + // scheduled instructions. + LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt); } + // If the block was split, make sure we update any references that are used to + // update PHI nodes later on. + if (FirstMBB != LastMBB) + SDB->UpdateSplitBlock(FirstMBB, LastMBB); + // Free the scheduler state. { NamedRegionTimer T("Instruction Scheduling Cleanup", GroupName, @@ -591,23 +784,48 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { CurDAG->clear(); } +namespace { +/// ISelUpdater - helper class to handle updates of the instruction selection +/// graph. +class ISelUpdater : public SelectionDAG::DAGUpdateListener { + SelectionDAG::allnodes_iterator &ISelPosition; +public: + ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp) + : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {} + + /// NodeDeleted - Handle nodes deleted from the graph. If the node being + /// deleted is the current ISelPosition node, update ISelPosition. + /// + virtual void NodeDeleted(SDNode *N, SDNode *E) { + if (ISelPosition == SelectionDAG::allnodes_iterator(N)) + ++ISelPosition; + } +}; +} // end anonymous namespace + void SelectionDAGISel::DoInstructionSelection() { - DEBUG(errs() << "===== Instruction selection begins:\n"); + DEBUG(dbgs() << "===== Instruction selection begins: BB#" + << FuncInfo->MBB->getNumber() + << " '" << FuncInfo->MBB->getName() << "'\n"); PreprocessISelDAG(); - + // Select target instructions for the DAG. { // Number all nodes with a topological order and set DAGSize. DAGSize = CurDAG->AssignTopologicalOrder(); - + // Create a dummy node (which is not added to allnodes), that adds // a reference to the root node, preventing it from being deleted, // and tracking any changes of the root. HandleSDNode Dummy(CurDAG->getRoot()); - ISelPosition = SelectionDAG::allnodes_iterator(CurDAG->getRoot().getNode()); + SelectionDAG::allnodes_iterator ISelPosition (CurDAG->getRoot().getNode()); ++ISelPosition; - + + // Make sure that ISelPosition gets properly updated when nodes are deleted + // in calls made from this function. + ISelUpdater ISU(*CurDAG, ISelPosition); + // The AllNodes list is now topological-sorted. Visit the // nodes by starting at the end of the list (the root of the // graph) and preceding back toward the beginning (the entry @@ -619,31 +837,30 @@ void SelectionDAGISel::DoInstructionSelection() { // makes it theoretically possible to disable the DAGCombiner. if (Node->use_empty()) continue; - + SDNode *ResNode = Select(Node); - + // FIXME: This is pretty gross. 'Select' should be changed to not return // anything at all and this code should be nuked with a tactical strike. - + // If node should not be replaced, continue with the next one. if (ResNode == Node || Node->getOpcode() == ISD::DELETED_NODE) continue; // Replace node. - if (ResNode) + if (ResNode) { ReplaceUses(Node, ResNode); - + } + // If after the replacement this node is not used any more, // remove this dead node. - if (Node->use_empty()) { // Don't delete EntryToken, etc. - ISelUpdater ISU(ISelPosition); - CurDAG->RemoveDeadNode(Node, &ISU); - } + if (Node->use_empty()) // Don't delete EntryToken, etc. + CurDAG->RemoveDeadNode(Node); } - + CurDAG->setRoot(Dummy.getValue()); - } + } - DEBUG(errs() << "===== Instruction selection ends:\n"); + DEBUG(dbgs() << "===== Instruction selection ends:\n"); PostprocessISelDAG(); } @@ -651,73 +868,169 @@ void SelectionDAGISel::DoInstructionSelection() { /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and /// do other setup for EH landing-pad blocks. void SelectionDAGISel::PrepareEHLandingPad() { + MachineBasicBlock *MBB = FuncInfo->MBB; + // Add a label to mark the beginning of the landing pad. Deletion of the // landing pad can thus be detected via the MachineModuleInfo. - MCSymbol *Label = MF->getMMI().addLandingPad(FuncInfo->MBB); + MCSymbol *Label = MF->getMMI().addLandingPad(MBB); - const TargetInstrDesc &II = TM.getInstrInfo()->get(TargetOpcode::EH_LABEL); - BuildMI(*FuncInfo->MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II) + // Assign the call site to the landing pad's begin label. + MF->getMMI().setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]); + + const MCInstrDesc &II = TM.getInstrInfo()->get(TargetOpcode::EH_LABEL); + BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II) .addSym(Label); // Mark exception register as live in. - unsigned Reg = TLI.getExceptionAddressRegister(); - if (Reg) FuncInfo->MBB->addLiveIn(Reg); + const TargetLowering *TLI = getTargetLowering(); + const TargetRegisterClass *PtrRC = TLI->getRegClassFor(TLI->getPointerTy()); + if (unsigned Reg = TLI->getExceptionPointerRegister()) + FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC); // Mark exception selector register as live in. - Reg = TLI.getExceptionSelectorRegister(); - if (Reg) FuncInfo->MBB->addLiveIn(Reg); - - // FIXME: Hack around an exception handling flaw (PR1508): the personality - // function and list of typeids logically belong to the invoke (or, if you - // like, the basic block containing the invoke), and need to be associated - // with it in the dwarf exception handling tables. Currently however the - // information is provided by an intrinsic (eh.selector) that can be moved - // to unexpected places by the optimizers: if the unwind edge is critical, - // then breaking it can result in the intrinsics being in the successor of - // the landing pad, not the landing pad itself. This results - // in exceptions not being caught because no typeids are associated with - // the invoke. This may not be the only way things can go wrong, but it - // is the only way we try to work around for the moment. - const BasicBlock *LLVMBB = FuncInfo->MBB->getBasicBlock(); - const BranchInst *Br = dyn_cast(LLVMBB->getTerminator()); - - if (Br && Br->isUnconditional()) { // Critical edge? - BasicBlock::const_iterator I, E; - for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I) - if (isa(I)) - break; + if (unsigned Reg = TLI->getExceptionSelectorRegister()) + FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC); +} + +/// isFoldedOrDeadInstruction - Return true if the specified instruction is +/// side-effect free and is either dead or folded into a generated instruction. +/// Return false if it needs to be emitted. +static bool isFoldedOrDeadInstruction(const Instruction *I, + FunctionLoweringInfo *FuncInfo) { + return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded. + !isa(I) && // Terminators aren't folded. + !isa(I) && // Debug instructions aren't folded. + !isa(I) && // Landingpad instructions aren't folded. + !FuncInfo->isExportedInst(I); // Exported instrs must be computed. +} - if (I == E) - // No catch info found - try to extract some from the successor. - CopyCatchInfo(Br->getSuccessor(0), LLVMBB, &MF->getMMI(), *FuncInfo); +#ifndef NDEBUG +// Collect per Instruction statistics for fast-isel misses. Only those +// instructions that cause the bail are accounted for. It does not account for +// instructions higher in the block. Thus, summing the per instructions stats +// will not add up to what is reported by NumFastIselFailures. +static void collectFailStats(const Instruction *I) { + switch (I->getOpcode()) { + default: assert (0 && " "); + + // Terminators + case Instruction::Ret: NumFastIselFailRet++; return; + case Instruction::Br: NumFastIselFailBr++; return; + case Instruction::Switch: NumFastIselFailSwitch++; return; + case Instruction::IndirectBr: NumFastIselFailIndirectBr++; return; + case Instruction::Invoke: NumFastIselFailInvoke++; return; + case Instruction::Resume: NumFastIselFailResume++; return; + case Instruction::Unreachable: NumFastIselFailUnreachable++; return; + + // Standard binary operators... + case Instruction::Add: NumFastIselFailAdd++; return; + case Instruction::FAdd: NumFastIselFailFAdd++; return; + case Instruction::Sub: NumFastIselFailSub++; return; + case Instruction::FSub: NumFastIselFailFSub++; return; + case Instruction::Mul: NumFastIselFailMul++; return; + case Instruction::FMul: NumFastIselFailFMul++; return; + case Instruction::UDiv: NumFastIselFailUDiv++; return; + case Instruction::SDiv: NumFastIselFailSDiv++; return; + case Instruction::FDiv: NumFastIselFailFDiv++; return; + case Instruction::URem: NumFastIselFailURem++; return; + case Instruction::SRem: NumFastIselFailSRem++; return; + case Instruction::FRem: NumFastIselFailFRem++; return; + + // Logical operators... + case Instruction::And: NumFastIselFailAnd++; return; + case Instruction::Or: NumFastIselFailOr++; return; + case Instruction::Xor: NumFastIselFailXor++; return; + + // Memory instructions... + case Instruction::Alloca: NumFastIselFailAlloca++; return; + case Instruction::Load: NumFastIselFailLoad++; return; + case Instruction::Store: NumFastIselFailStore++; return; + case Instruction::AtomicCmpXchg: NumFastIselFailAtomicCmpXchg++; return; + case Instruction::AtomicRMW: NumFastIselFailAtomicRMW++; return; + case Instruction::Fence: NumFastIselFailFence++; return; + case Instruction::GetElementPtr: NumFastIselFailGetElementPtr++; return; + + // Convert instructions... + case Instruction::Trunc: NumFastIselFailTrunc++; return; + case Instruction::ZExt: NumFastIselFailZExt++; return; + case Instruction::SExt: NumFastIselFailSExt++; return; + case Instruction::FPTrunc: NumFastIselFailFPTrunc++; return; + case Instruction::FPExt: NumFastIselFailFPExt++; return; + case Instruction::FPToUI: NumFastIselFailFPToUI++; return; + case Instruction::FPToSI: NumFastIselFailFPToSI++; return; + case Instruction::UIToFP: NumFastIselFailUIToFP++; return; + case Instruction::SIToFP: NumFastIselFailSIToFP++; return; + case Instruction::IntToPtr: NumFastIselFailIntToPtr++; return; + case Instruction::PtrToInt: NumFastIselFailPtrToInt++; return; + case Instruction::BitCast: NumFastIselFailBitCast++; return; + + // Other instructions... + case Instruction::ICmp: NumFastIselFailICmp++; return; + case Instruction::FCmp: NumFastIselFailFCmp++; return; + case Instruction::PHI: NumFastIselFailPHI++; return; + case Instruction::Select: NumFastIselFailSelect++; return; + case Instruction::Call: NumFastIselFailCall++; return; + case Instruction::Shl: NumFastIselFailShl++; return; + case Instruction::LShr: NumFastIselFailLShr++; return; + case Instruction::AShr: NumFastIselFailAShr++; return; + case Instruction::VAArg: NumFastIselFailVAArg++; return; + case Instruction::ExtractElement: NumFastIselFailExtractElement++; return; + case Instruction::InsertElement: NumFastIselFailInsertElement++; return; + case Instruction::ShuffleVector: NumFastIselFailShuffleVector++; return; + case Instruction::ExtractValue: NumFastIselFailExtractValue++; return; + case Instruction::InsertValue: NumFastIselFailInsertValue++; return; + case Instruction::LandingPad: NumFastIselFailLandingPad++; return; } } +#endif void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // Initialize the Fast-ISel state, if needed. FastISel *FastIS = 0; - if (EnableFastISel) - FastIS = TLI.createFastISel(*FuncInfo); + if (TM.Options.EnableFastISel || Fn.hasFnAttribute(Attribute::OptimizeNone)) + FastIS = getTargetLowering()->createFastISel(*FuncInfo, LibInfo); // Iterate over all basic blocks in the function. - for (Function::const_iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { - const BasicBlock *LLVMBB = &*I; - FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB]; - FuncInfo->InsertPt = FuncInfo->MBB->getFirstNonPHI(); + ReversePostOrderTraversal RPOT(&Fn); + for (ReversePostOrderTraversal::rpo_iterator + I = RPOT.begin(), E = RPOT.end(); I != E; ++I) { + const BasicBlock *LLVMBB = *I; + + if (OptLevel != CodeGenOpt::None) { + bool AllPredsVisited = true; + for (const_pred_iterator PI = pred_begin(LLVMBB), PE = pred_end(LLVMBB); + PI != PE; ++PI) { + if (!FuncInfo->VisitedBBs.count(*PI)) { + AllPredsVisited = false; + break; + } + } + + if (AllPredsVisited) { + for (BasicBlock::const_iterator I = LLVMBB->begin(); + const PHINode *PN = dyn_cast(I); ++I) + FuncInfo->ComputePHILiveOutRegInfo(PN); + } else { + for (BasicBlock::const_iterator I = LLVMBB->begin(); + const PHINode *PN = dyn_cast(I); ++I) + FuncInfo->InvalidatePHILiveOutRegInfo(PN); + } + + FuncInfo->VisitedBBs.insert(LLVMBB); + } BasicBlock::const_iterator const Begin = LLVMBB->getFirstNonPHI(); BasicBlock::const_iterator const End = LLVMBB->end(); BasicBlock::const_iterator BI = End; + FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB]; FuncInfo->InsertPt = FuncInfo->MBB->getFirstNonPHI(); // Setup an EH landing-pad block. + FuncInfo->ExceptionPointerVirtReg = 0; + FuncInfo->ExceptionSelectorVirtReg = 0; if (FuncInfo->MBB->isLandingPad()) PrepareEHLandingPad(); - - // Lower any arguments needed in this block if this is the entry block. - if (LLVMBB == &Fn.getEntryBlock()) - LowerArguments(LLVMBB); // Before doing SelectionDAG ISel, see if FastISel has been requested. if (FastIS) { @@ -726,9 +1039,21 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // Emit code for any incoming arguments. This must happen before // beginning FastISel on the entry block. if (LLVMBB == &Fn.getEntryBlock()) { - CurDAG->setRoot(SDB->getControlRoot()); - SDB->clear(); - CodeGenAndEmitDAG(); + ++NumEntryBlocks; + + // Lower any arguments needed in this block if this is the entry block. + if (!FastIS->LowerArguments()) { + // Fast isel failed to lower these arguments + ++NumFastIselFailLowerArguments; + if (EnableFastISelAbortArgs) + llvm_unreachable("FastISel didn't lower all arguments"); + + // Use SelectionDAG argument lowering + LowerArguments(Fn); + CurDAG->setRoot(SDB->getControlRoot()); + SDB->clear(); + CodeGenAndEmitDAG(); + } // If we inserted any instructions at the beginning, make a note of // where they are, so we can be sure to emit subsequent instructions @@ -739,28 +1064,53 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { FastIS->setLastLocalValue(0); } + unsigned NumFastIselRemaining = std::distance(Begin, End); // Do FastISel on as many instructions as possible. for (; BI != Begin; --BI) { const Instruction *Inst = llvm::prior(BI); // If we no longer require this instruction, skip it. - if (!Inst->mayWriteToMemory() && - !isa(Inst) && - !isa(Inst) && - !FuncInfo->isExportedInst(Inst)) + if (isFoldedOrDeadInstruction(Inst, FuncInfo)) { + --NumFastIselRemaining; continue; + } // Bottom-up: reset the insert pos at the top, after any local-value // instructions. FastIS->recomputeInsertPt(); // Try to select the instruction with FastISel. - if (FastIS->SelectInstruction(Inst)) + if (FastIS->SelectInstruction(Inst)) { + --NumFastIselRemaining; + ++NumFastIselSuccess; + // If fast isel succeeded, skip over all the folded instructions, and + // then see if there is a load right before the selected instructions. + // Try to fold the load if so. + const Instruction *BeforeInst = Inst; + while (BeforeInst != Begin) { + BeforeInst = llvm::prior(BasicBlock::const_iterator(BeforeInst)); + if (!isFoldedOrDeadInstruction(BeforeInst, FuncInfo)) + break; + } + if (BeforeInst != Inst && isa(BeforeInst) && + BeforeInst->hasOneUse() && + FastIS->tryToFoldLoad(cast(BeforeInst), Inst)) { + // If we succeeded, don't re-select the load. + BI = llvm::next(BasicBlock::const_iterator(BeforeInst)); + --NumFastIselRemaining; + ++NumFastIselSuccess; + } continue; + } + +#ifndef NDEBUG + if (EnableFastISelVerbose2) + collectFailStats(Inst); +#endif // Then handle certain instructions as single-LLVM-Instruction blocks. if (isa(Inst)) { - ++NumFastIselFailures; + if (EnableFastISelVerbose || EnableFastISelAbort) { dbgs() << "FastISel missed call: "; Inst->dump(); @@ -773,21 +1123,34 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { } bool HadTailCall = false; + MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt; SelectBasicBlock(Inst, BI, HadTailCall); // If the call was emitted as a tail call, we're done with the block. + // We also need to delete any previously emitted instructions. if (HadTailCall) { + FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end()); --BI; break; } + // Recompute NumFastIselRemaining as Selection DAG instruction + // selection may have handled the call, input args, etc. + unsigned RemainingNow = std::distance(Begin, BI); + NumFastIselFailures += NumFastIselRemaining - RemainingNow; + NumFastIselRemaining = RemainingNow; continue; } - // Otherwise, give up on FastISel for the rest of the block. - // For now, be a little lenient about non-branch terminators. - if (!isa(Inst) || isa(Inst)) { - ++NumFastIselFailures; + if (isa(Inst) && !isa(Inst)) { + // Don't abort, and use a different message for terminator misses. + NumFastIselFailures += NumFastIselRemaining; + if (EnableFastISelVerbose || EnableFastISelAbort) { + dbgs() << "FastISel missed terminator: "; + Inst->dump(); + } + } else { + NumFastIselFailures += NumFastIselRemaining; if (EnableFastISelVerbose || EnableFastISelAbort) { dbgs() << "FastISel miss: "; Inst->dump(); @@ -801,19 +1164,118 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { } FastIS->recomputeInsertPt(); + } else { + // Lower any arguments needed in this block if this is the entry block. + if (LLVMBB == &Fn.getEntryBlock()) { + ++NumEntryBlocks; + LowerArguments(Fn); + } } - // Run SelectionDAG instruction selection on the remainder of the block - // not handled by FastISel. If FastISel is not run, this is the entire - // block. - bool HadTailCall; - SelectBasicBlock(Begin, BI, HadTailCall); + if (Begin != BI) + ++NumDAGBlocks; + else + ++NumFastIselBlocks; + + if (Begin != BI) { + // Run SelectionDAG instruction selection on the remainder of the block + // not handled by FastISel. If FastISel is not run, this is the entire + // block. + bool HadTailCall; + SelectBasicBlock(Begin, BI, HadTailCall); + } FinishBasicBlock(); FuncInfo->PHINodesToUpdate.clear(); } delete FastIS; + SDB->clearDanglingDebugInfo(); + SDB->SPDescriptor.resetPerFunctionState(); +} + +/// Given that the input MI is before a partial terminator sequence TSeq, return +/// true if M + TSeq also a partial terminator sequence. +/// +/// A Terminator sequence is a sequence of MachineInstrs which at this point in +/// lowering copy vregs into physical registers, which are then passed into +/// terminator instructors so we can satisfy ABI constraints. A partial +/// terminator sequence is an improper subset of a terminator sequence (i.e. it +/// may be the whole terminator sequence). +static bool MIIsInTerminatorSequence(const MachineInstr *MI) { + // If we do not have a copy or an implicit def, we return true if and only if + // MI is a debug value. + if (!MI->isCopy() && !MI->isImplicitDef()) + // Sometimes DBG_VALUE MI sneak in between the copies from the vregs to the + // physical registers if there is debug info associated with the terminator + // of our mbb. We want to include said debug info in our terminator + // sequence, so we return true in that case. + return MI->isDebugValue(); + + // We have left the terminator sequence if we are not doing one of the + // following: + // + // 1. Copying a vreg into a physical register. + // 2. Copying a vreg into a vreg. + // 3. Defining a register via an implicit def. + + // OPI should always be a register definition... + MachineInstr::const_mop_iterator OPI = MI->operands_begin(); + if (!OPI->isReg() || !OPI->isDef()) + return false; + + // Defining any register via an implicit def is always ok. + if (MI->isImplicitDef()) + return true; + + // Grab the copy source... + MachineInstr::const_mop_iterator OPI2 = OPI; + ++OPI2; + assert(OPI2 != MI->operands_end() + && "Should have a copy implying we should have 2 arguments."); + + // Make sure that the copy dest is not a vreg when the copy source is a + // physical register. + if (!OPI2->isReg() || + (!TargetRegisterInfo::isPhysicalRegister(OPI->getReg()) && + TargetRegisterInfo::isPhysicalRegister(OPI2->getReg()))) + return false; + + return true; +} + +/// Find the split point at which to splice the end of BB into its success stack +/// protector check machine basic block. +/// +/// On many platforms, due to ABI constraints, terminators, even before register +/// allocation, use physical registers. This creates an issue for us since +/// physical registers at this point can not travel across basic +/// blocks. Luckily, selectiondag always moves physical registers into vregs +/// when they enter functions and moves them through a sequence of copies back +/// into the physical registers right before the terminator creating a +/// ``Terminator Sequence''. This function is searching for the beginning of the +/// terminator sequence so that we can ensure that we splice off not just the +/// terminator, but additionally the copies that move the vregs into the +/// physical registers. +static MachineBasicBlock::iterator +FindSplitPointForStackProtector(MachineBasicBlock *BB, DebugLoc DL) { + MachineBasicBlock::iterator SplitPoint = BB->getFirstTerminator(); + // + if (SplitPoint == BB->begin()) + return SplitPoint; + + MachineBasicBlock::iterator Start = BB->begin(); + MachineBasicBlock::iterator Previous = SplitPoint; + --Previous; + + while (MIIsInTerminatorSequence(Previous)) { + SplitPoint = Previous; + if (Previous == Start) + break; + --Previous; + } + + return SplitPoint; } void @@ -826,24 +1288,69 @@ SelectionDAGISel::FinishBasicBlock() { << FuncInfo->PHINodesToUpdate[i].first << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n"); + const bool MustUpdatePHINodes = SDB->SwitchCases.empty() && + SDB->JTCases.empty() && + SDB->BitTestCases.empty(); + // Next, now that we know what the last MBB the LLVM BB expanded is, update // PHI nodes in successors. - if (SDB->SwitchCases.empty() && - SDB->JTCases.empty() && - SDB->BitTestCases.empty()) { + if (MustUpdatePHINodes) { for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { - MachineInstr *PHI = FuncInfo->PHINodesToUpdate[i].first; + MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); if (!FuncInfo->MBB->isSuccessor(PHI->getParent())) continue; - PHI->addOperand( - MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[i].second, false)); - PHI->addOperand(MachineOperand::CreateMBB(FuncInfo->MBB)); + PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB); } - return; } + // Handle stack protector. + if (SDB->SPDescriptor.shouldEmitStackProtector()) { + MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB(); + MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB(); + + // Find the split point to split the parent mbb. At the same time copy all + // physical registers used in the tail of parent mbb into virtual registers + // before the split point and back into physical registers after the split + // point. This prevents us needing to deal with Live-ins and many other + // register allocation issues caused by us splitting the parent mbb. The + // register allocator will clean up said virtual copies later on. + MachineBasicBlock::iterator SplitPoint = + FindSplitPointForStackProtector(ParentMBB, SDB->getCurDebugLoc()); + + // Splice the terminator of ParentMBB into SuccessMBB. + SuccessMBB->splice(SuccessMBB->end(), ParentMBB, + SplitPoint, + ParentMBB->end()); + + // Add compare/jump on neq/jump to the parent BB. + FuncInfo->MBB = ParentMBB; + FuncInfo->InsertPt = ParentMBB->end(); + SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB); + CurDAG->setRoot(SDB->getRoot()); + SDB->clear(); + CodeGenAndEmitDAG(); + + // CodeGen Failure MBB if we have not codegened it yet. + MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB(); + if (!FailureMBB->size()) { + FuncInfo->MBB = FailureMBB; + FuncInfo->InsertPt = FailureMBB->end(); + SDB->visitSPDescriptorFailure(SDB->SPDescriptor); + CurDAG->setRoot(SDB->getRoot()); + SDB->clear(); + CodeGenAndEmitDAG(); + } + + // Clear the Per-BB State. + SDB->SPDescriptor.resetPerBBState(); + } + + // If we updated PHI Nodes, return early. + if (MustUpdatePHINodes) + return; + for (unsigned i = 0, e = SDB->BitTestCases.size(); i != e; ++i) { // Lower header first, if it wasn't already lowered if (!SDB->BitTestCases[i].Emitted) { @@ -857,18 +1364,27 @@ SelectionDAGISel::FinishBasicBlock() { CodeGenAndEmitDAG(); } + uint32_t UnhandledWeight = 0; + for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) + UnhandledWeight += SDB->BitTestCases[i].Cases[j].ExtraWeight; + for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) { + UnhandledWeight -= SDB->BitTestCases[i].Cases[j].ExtraWeight; // Set the current basic block to the mbb we wish to insert the code into FuncInfo->MBB = SDB->BitTestCases[i].Cases[j].ThisBB; FuncInfo->InsertPt = FuncInfo->MBB->end(); // Emit the code if (j+1 != ej) - SDB->visitBitTestCase(SDB->BitTestCases[i].Cases[j+1].ThisBB, + SDB->visitBitTestCase(SDB->BitTestCases[i], + SDB->BitTestCases[i].Cases[j+1].ThisBB, + UnhandledWeight, SDB->BitTestCases[i].Reg, SDB->BitTestCases[i].Cases[j], FuncInfo->MBB); else - SDB->visitBitTestCase(SDB->BitTestCases[i].Default, + SDB->visitBitTestCase(SDB->BitTestCases[i], + SDB->BitTestCases[i].Default, + UnhandledWeight, SDB->BitTestCases[i].Reg, SDB->BitTestCases[i].Cases[j], FuncInfo->MBB); @@ -882,33 +1398,23 @@ SelectionDAGISel::FinishBasicBlock() { // Update PHI Nodes for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); pi != pe; ++pi) { - MachineInstr *PHI = FuncInfo->PHINodesToUpdate[pi].first; + MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first); MachineBasicBlock *PHIBB = PHI->getParent(); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); // This is "default" BB. We have two jumps to it. From "header" BB and // from last "case" BB. - if (PHIBB == SDB->BitTestCases[i].Default) { - PHI->addOperand(MachineOperand:: - CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand(MachineOperand::CreateMBB(SDB->BitTestCases[i].Parent)); - PHI->addOperand(MachineOperand:: - CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand(MachineOperand::CreateMBB(SDB->BitTestCases[i].Cases. - back().ThisBB)); - } + if (PHIBB == SDB->BitTestCases[i].Default) + PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second) + .addMBB(SDB->BitTestCases[i].Parent) + .addReg(FuncInfo->PHINodesToUpdate[pi].second) + .addMBB(SDB->BitTestCases[i].Cases.back().ThisBB); // One of "cases" BB. for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) { MachineBasicBlock* cBB = SDB->BitTestCases[i].Cases[j].ThisBB; - if (cBB->isSuccessor(PHIBB)) { - PHI->addOperand(MachineOperand:: - CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand(MachineOperand::CreateMBB(cBB)); - } + if (cBB->isSuccessor(PHIBB)) + PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB); } } } @@ -943,25 +1449,17 @@ SelectionDAGISel::FinishBasicBlock() { // Update PHI Nodes for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); pi != pe; ++pi) { - MachineInstr *PHI = FuncInfo->PHINodesToUpdate[pi].first; + MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first); MachineBasicBlock *PHIBB = PHI->getParent(); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); // "default" BB. We can go there only from header BB. - if (PHIBB == SDB->JTCases[i].second.Default) { - PHI->addOperand - (MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand - (MachineOperand::CreateMBB(SDB->JTCases[i].first.HeaderBB)); - } + if (PHIBB == SDB->JTCases[i].second.Default) + PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second) + .addMBB(SDB->JTCases[i].first.HeaderBB); // JT BB. Just iterate over successors here - if (FuncInfo->MBB->isSuccessor(PHIBB)) { - PHI->addOperand - (MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand(MachineOperand::CreateMBB(FuncInfo->MBB)); - } + if (FuncInfo->MBB->isSuccessor(PHIBB)) + PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB); } } SDB->JTCases.clear(); @@ -969,21 +1467,18 @@ SelectionDAGISel::FinishBasicBlock() { // If the switch block involved a branch to one of the actual successors, we // need to update PHI nodes in that block. for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { - MachineInstr *PHI = FuncInfo->PHINodesToUpdate[i].first; + MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); - if (FuncInfo->MBB->isSuccessor(PHI->getParent())) { - PHI->addOperand( - MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[i].second, false)); - PHI->addOperand(MachineOperand::CreateMBB(FuncInfo->MBB)); - } + if (FuncInfo->MBB->isSuccessor(PHI->getParent())) + PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB); } // If we generated any switch lowering information, build and codegen any // additional DAGs necessary. for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) { // Set the current basic block to the mbb we wish to insert the code into - MachineBasicBlock *ThisBB = FuncInfo->MBB = SDB->SwitchCases[i].ThisBB; + FuncInfo->MBB = SDB->SwitchCases[i].ThisBB; FuncInfo->InsertPt = FuncInfo->MBB->end(); // Determine the unique successors. @@ -992,13 +1487,15 @@ SelectionDAGISel::FinishBasicBlock() { if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB) Succs.push_back(SDB->SwitchCases[i].FalseBB); - // Emit the code. Note that this could result in ThisBB being split, so - // we need to check for updates. + // Emit the code. Note that this could result in FuncInfo->MBB being split. SDB->visitSwitchCase(SDB->SwitchCases[i], FuncInfo->MBB); CurDAG->setRoot(SDB->getRoot()); SDB->clear(); CodeGenAndEmitDAG(); - ThisBB = FuncInfo->MBB; + + // Remember the last block, now that any splitting is done, for use in + // populating PHI nodes in successors. + MachineBasicBlock *ThisBB = FuncInfo->MBB; // Handle any PHI nodes in successors of this chunk, as if we were coming // from the original BB before switch expansion. Note that PHI nodes can @@ -1010,18 +1507,16 @@ SelectionDAGISel::FinishBasicBlock() { // FuncInfo->MBB may have been removed from the CFG if a branch was // constant folded. if (ThisBB->isSuccessor(FuncInfo->MBB)) { - for (MachineBasicBlock::iterator Phi = FuncInfo->MBB->begin(); - Phi != FuncInfo->MBB->end() && Phi->isPHI(); - ++Phi) { + for (MachineBasicBlock::iterator + MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end(); + MBBI != MBBE && MBBI->isPHI(); ++MBBI) { + MachineInstrBuilder PHI(*MF, MBBI); // This value for this PHI node is recorded in PHINodesToUpdate. for (unsigned pn = 0; ; ++pn) { assert(pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"); - if (FuncInfo->PHINodesToUpdate[pn].first == Phi) { - Phi->addOperand(MachineOperand:: - CreateReg(FuncInfo->PHINodesToUpdate[pn].second, - false)); - Phi->addOperand(MachineOperand::CreateMBB(ThisBB)); + if (FuncInfo->PHINodesToUpdate[pn].first == PHI) { + PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB); break; } } @@ -1048,10 +1543,6 @@ ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() { return Ctor(this, OptLevel); } -ScheduleHazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() { - return new ScheduleHazardRecognizer(); -} - //===----------------------------------------------------------------------===// // Helper functions used by the generated instruction selector. //===----------------------------------------------------------------------===// @@ -1108,7 +1599,7 @@ bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS, APInt NeededMask = DesiredMask & ~ActualMask; APInt KnownZero, KnownOne; - CurDAG->ComputeMaskedBits(LHS, NeededMask, KnownZero, KnownOne); + CurDAG->ComputeMaskedBits(LHS, KnownZero, KnownOne); // If all the missing bits in the or are already known to be set, match! if ((NeededMask & KnownOne) == NeededMask) @@ -1131,11 +1622,11 @@ SelectInlineAsmMemoryOperands(std::vector &Ops) { Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0 Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1 Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc - Ops.push_back(InOps[InlineAsm::Op_IsAlignStack]); // 3 + Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack) unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size(); - if (InOps[e-1].getValueType() == MVT::Flag) - --e; // Don't process a flag operand if it is here. + if (InOps[e-1].getValueType() == MVT::Glue) + --e; // Don't process a glue operand if it is here. while (i != e) { unsigned Flags = cast(InOps[i])->getZExtValue(); @@ -1162,15 +1653,15 @@ SelectInlineAsmMemoryOperands(std::vector &Ops) { } } - // Add the flag input back if present. + // Add the glue input back if present. if (e != InOps.size()) Ops.push_back(InOps.back()); } -/// findFlagUse - Return use of EVT::Flag value produced by the specified +/// findGlueUse - Return use of MVT::Glue value produced by the specified /// SDNode. /// -static SDNode *findFlagUse(SDNode *N) { +static SDNode *findGlueUse(SDNode *N) { unsigned FlagResNo = N->getNumValues()-1; for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { SDUse &Use = I.getUse(); @@ -1192,11 +1683,11 @@ static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse, // never find it. // // The Use may be -1 (unassigned) if it is a newly allocated node. This can - // happen because we scan down to newly selected nodes in the case of flag + // happen because we scan down to newly selected nodes in the case of glue // uses. if ((Use->getNodeId() < Def->getNodeId() && Use->getNodeId() != -1)) return false; - + // Don't revisit nodes if we already scanned it and didn't fail, we know we // won't fail if we scan it again. if (!Visited.insert(Use)) @@ -1206,7 +1697,7 @@ static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse, // Ignore chain uses, they are validated by HandleMergeInputChains. if (Use->getOperand(i).getValueType() == MVT::Other && IgnoreChains) continue; - + SDNode *N = Use->getOperand(i).getNode(); if (N == Def) { if (Use == ImmedUse || Use == Root) @@ -1253,8 +1744,8 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, // // * indicates nodes to be folded together. // - // If Root produces a flag, then it gets (even more) interesting. Since it - // will be "glued" together with its flag use in the scheduler, we need to + // If Root produces glue, then it gets (even more) interesting. Since it + // will be "glued" together with its glue use in the scheduler, we need to // check if it might reach N. // // [N*] // @@ -1272,30 +1763,30 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, // ^ / // // f / // // | / // - // [FU] // + // [GU] // // - // If FU (flag use) indirectly reaches N (the load), and Root folds N - // (call it Fold), then X is a predecessor of FU and a successor of - // Fold. But since Fold and FU are flagged together, this will create + // If GU (glue use) indirectly reaches N (the load), and Root folds N + // (call it Fold), then X is a predecessor of GU and a successor of + // Fold. But since Fold and GU are glued together, this will create // a cycle in the scheduling graph. - // If the node has flags, walk down the graph to the "lowest" node in the - // flagged set. + // If the node has glue, walk down the graph to the "lowest" node in the + // glueged set. EVT VT = Root->getValueType(Root->getNumValues()-1); - while (VT == MVT::Flag) { - SDNode *FU = findFlagUse(Root); - if (FU == NULL) + while (VT == MVT::Glue) { + SDNode *GU = findGlueUse(Root); + if (GU == NULL) break; - Root = FU; + Root = GU; VT = Root->getValueType(Root->getNumValues()-1); - - // If our query node has a flag result with a use, we've walked up it. If + + // If our query node has a glue result with a use, we've walked up it. If // the user (which has already been selected) has a chain or indirectly uses // the chain, our WalkChainUsers predicate will not consider it. Because of // this, we cannot ignore chains in this predicate. IgnoreChains = false; } - + SmallPtrSet Visited; return !findNonImmUse(Root, N.getNode(), U, Root, Visited, IgnoreChains); @@ -1304,11 +1795,9 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, SDNode *SelectionDAGISel::Select_INLINEASM(SDNode *N) { std::vector Ops(N->op_begin(), N->op_end()); SelectInlineAsmMemoryOperands(Ops); - - std::vector VTs; - VTs.push_back(MVT::Other); - VTs.push_back(MVT::Flag); - SDValue New = CurDAG->getNode(ISD::INLINEASM, N->getDebugLoc(), + + EVT VTs[] = { MVT::Other, MVT::Glue }; + SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), VTs, &Ops[0], Ops.size()); New->setNodeId(-1); return New.getNode(); @@ -1319,11 +1808,11 @@ SDNode *SelectionDAGISel::Select_UNDEF(SDNode *N) { } /// GetVBR - decode a vbr encoding whose top bit is set. -ALWAYS_INLINE static uint64_t +LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) { assert(Val >= 128 && "Not a VBR"); Val &= 127; // Remove first vbr bit. - + unsigned Shift = 7; uint64_t NextBits; do { @@ -1331,25 +1820,23 @@ GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) { Val |= (NextBits&127) << Shift; Shift += 7; } while (NextBits & 128); - + return Val; } -/// UpdateChainsAndFlags - When a match is complete, this method updates uses of -/// interior flag and chain results to use the new flag and chain results. +/// UpdateChainsAndGlue - When a match is complete, this method updates uses of +/// interior glue and chain results to use the new glue and chain results. void SelectionDAGISel:: -UpdateChainsAndFlags(SDNode *NodeToMatch, SDValue InputChain, - const SmallVectorImpl &ChainNodesMatched, - SDValue InputFlag, - const SmallVectorImpl &FlagResultNodesMatched, - bool isMorphNodeTo) { +UpdateChainsAndGlue(SDNode *NodeToMatch, SDValue InputChain, + const SmallVectorImpl &ChainNodesMatched, + SDValue InputGlue, + const SmallVectorImpl &GlueResultNodesMatched, + bool isMorphNodeTo) { SmallVector NowDeadNodes; - - ISelUpdater ISU(ISelPosition); // Now that all the normal results are replaced, we replace the chain and - // flag results if present. + // glue results if present. if (!ChainNodesMatched.empty()) { assert(InputChain.getNode() != 0 && "Matched input chains but didn't produce a chain"); @@ -1357,56 +1844,56 @@ UpdateChainsAndFlags(SDNode *NodeToMatch, SDValue InputChain, // Replace all the chain results with the final chain we ended up with. for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) { SDNode *ChainNode = ChainNodesMatched[i]; - + // If this node was already deleted, don't look at it. if (ChainNode->getOpcode() == ISD::DELETED_NODE) continue; - + // Don't replace the results of the root node if we're doing a // MorphNodeTo. if (ChainNode == NodeToMatch && isMorphNodeTo) continue; - + SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1); - if (ChainVal.getValueType() == MVT::Flag) + if (ChainVal.getValueType() == MVT::Glue) ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2); assert(ChainVal.getValueType() == MVT::Other && "Not a chain?"); - CurDAG->ReplaceAllUsesOfValueWith(ChainVal, InputChain, &ISU); - + CurDAG->ReplaceAllUsesOfValueWith(ChainVal, InputChain); + // If the node became dead and we haven't already seen it, delete it. if (ChainNode->use_empty() && !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode)) NowDeadNodes.push_back(ChainNode); } } - - // If the result produces a flag, update any flag results in the matched - // pattern with the flag result. - if (InputFlag.getNode() != 0) { + + // If the result produces glue, update any glue results in the matched + // pattern with the glue result. + if (InputGlue.getNode() != 0) { // Handle any interior nodes explicitly marked. - for (unsigned i = 0, e = FlagResultNodesMatched.size(); i != e; ++i) { - SDNode *FRN = FlagResultNodesMatched[i]; - + for (unsigned i = 0, e = GlueResultNodesMatched.size(); i != e; ++i) { + SDNode *FRN = GlueResultNodesMatched[i]; + // If this node was already deleted, don't look at it. if (FRN->getOpcode() == ISD::DELETED_NODE) continue; - - assert(FRN->getValueType(FRN->getNumValues()-1) == MVT::Flag && - "Doesn't have a flag result"); + + assert(FRN->getValueType(FRN->getNumValues()-1) == MVT::Glue && + "Doesn't have a glue result"); CurDAG->ReplaceAllUsesOfValueWith(SDValue(FRN, FRN->getNumValues()-1), - InputFlag, &ISU); - + InputGlue); + // If the node became dead and we haven't already seen it, delete it. if (FRN->use_empty() && !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), FRN)) NowDeadNodes.push_back(FRN); } } - + if (!NowDeadNodes.empty()) - CurDAG->RemoveDeadNodes(NowDeadNodes, &ISU); - - DEBUG(errs() << "ISEL: Match complete!\n"); + CurDAG->RemoveDeadNodes(NowDeadNodes); + + DEBUG(dbgs() << "ISEL: Match complete!\n"); } enum ChainResult { @@ -1424,30 +1911,33 @@ enum ChainResult { /// /// The walk we do here is guaranteed to be small because we quickly get down to /// already selected nodes "below" us. -static ChainResult -WalkChainUsers(SDNode *ChainedNode, +static ChainResult +WalkChainUsers(const SDNode *ChainedNode, SmallVectorImpl &ChainedNodesInPattern, SmallVectorImpl &InteriorChainedNodes) { ChainResult Result = CR_Simple; - + for (SDNode::use_iterator UI = ChainedNode->use_begin(), E = ChainedNode->use_end(); UI != E; ++UI) { // Make sure the use is of the chain, not some other value we produce. if (UI.getUse().getValueType() != MVT::Other) continue; - + SDNode *User = *UI; + if (User->getOpcode() == ISD::HANDLENODE) // Root of the graph. + continue; + // If we see an already-selected machine node, then we've gone beyond the // pattern that we're selecting down into the already selected chunk of the // DAG. + unsigned UserOpcode = User->getOpcode(); if (User->isMachineOpcode() || - User->getOpcode() == ISD::HANDLENODE) // Root of the graph. - continue; - - if (User->getOpcode() == ISD::CopyToReg || - User->getOpcode() == ISD::CopyFromReg || - User->getOpcode() == ISD::INLINEASM || - User->getOpcode() == ISD::EH_LABEL) { + UserOpcode == ISD::CopyToReg || + UserOpcode == ISD::CopyFromReg || + UserOpcode == ISD::INLINEASM || + UserOpcode == ISD::EH_LABEL || + UserOpcode == ISD::LIFETIME_START || + UserOpcode == ISD::LIFETIME_END) { // If their node ID got reset to -1 then they've already been selected. // Treat them like a MachineOpcode. if (User->getNodeId() == -1) @@ -1469,7 +1959,7 @@ WalkChainUsers(SDNode *ChainedNode, if (!std::count(ChainedNodesInPattern.begin(), ChainedNodesInPattern.end(), User)) return CR_InducesCycle; - + // Otherwise we found a node that is part of our pattern. For example in: // x = load ptr // y = x+4 @@ -1481,7 +1971,7 @@ WalkChainUsers(SDNode *ChainedNode, InteriorChainedNodes.push_back(User); continue; } - + // If we found a TokenFactor, there are two cases to consider: first if the // TokenFactor is just hanging "below" the pattern we're matching (i.e. no // uses of the TF are in our pattern) we just want to ignore it. Second, @@ -1518,7 +2008,7 @@ WalkChainUsers(SDNode *ChainedNode, case CR_LeadsToInteriorNode: break; // Otherwise, keep processing. } - + // Okay, we know we're in the interesting interior case. The TokenFactor // is now going to be considered part of the pattern so that we rewrite its // uses (it may have uses that are not part of the pattern) with the @@ -1529,7 +2019,7 @@ WalkChainUsers(SDNode *ChainedNode, InteriorChainedNodes.push_back(User); continue; } - + return Result; } @@ -1551,7 +2041,7 @@ HandleMergeInputChains(SmallVectorImpl &ChainNodesMatched, InteriorChainedNodes) == CR_InducesCycle) return SDValue(); // Would induce a cycle. } - + // Okay, we have walked all the matched nodes and collected TokenFactor nodes // that we are interested in. Form our input TokenFactor node. SmallVector InputChains; @@ -1562,14 +2052,14 @@ HandleMergeInputChains(SmallVectorImpl &ChainNodesMatched, if (N->getOpcode() != ISD::TokenFactor) { if (std::count(InteriorChainedNodes.begin(),InteriorChainedNodes.end(),N)) continue; - + // Otherwise, add the input chain. SDValue InChain = ChainNodesMatched[i]->getOperand(0); assert(InChain.getValueType() == MVT::Other && "Not a chain"); InputChains.push_back(InChain); continue; } - + // If we have a token factor, we want to add all inputs of the token factor // that are not part of the pattern we're matching. for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { @@ -1578,13 +2068,12 @@ HandleMergeInputChains(SmallVectorImpl &ChainNodesMatched, InputChains.push_back(N->getOperand(op)); } } - - SDValue Res; + if (InputChains.size() == 1) return InputChains[0]; - return CurDAG->getNode(ISD::TokenFactor, ChainNodesMatched[0]->getDebugLoc(), + return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]), MVT::Other, &InputChains[0], InputChains.size()); -} +} /// MorphNode - Handle morphing a node in place for the selector. SDNode *SelectionDAGISel:: @@ -1592,15 +2081,15 @@ MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, const SDValue *Ops, unsigned NumOps, unsigned EmitNodeInfo) { // It is possible we're using MorphNodeTo to replace a node with no // normal results with one that has a normal result (or we could be - // adding a chain) and the input could have flags and chains as well. + // adding a chain) and the input could have glue and chains as well. // In this case we need to shift the operands down. // FIXME: This is a horrible hack and broken in obscure cases, no worse // than the old isel though. - int OldFlagResultNo = -1, OldChainResultNo = -1; + int OldGlueResultNo = -1, OldChainResultNo = -1; unsigned NTMNumResults = Node->getNumValues(); - if (Node->getValueType(NTMNumResults-1) == MVT::Flag) { - OldFlagResultNo = NTMNumResults-1; + if (Node->getValueType(NTMNumResults-1) == MVT::Glue) { + OldGlueResultNo = NTMNumResults-1; if (NTMNumResults != 1 && Node->getValueType(NTMNumResults-2) == MVT::Other) OldChainResultNo = NTMNumResults-2; @@ -1621,54 +2110,67 @@ MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, } unsigned ResNumResults = Res->getNumValues(); - // Move the flag if needed. - if ((EmitNodeInfo & OPFL_FlagOutput) && OldFlagResultNo != -1 && - (unsigned)OldFlagResultNo != ResNumResults-1) - CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldFlagResultNo), + // Move the glue if needed. + if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 && + (unsigned)OldGlueResultNo != ResNumResults-1) + CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldGlueResultNo), SDValue(Res, ResNumResults-1)); - if ((EmitNodeInfo & OPFL_FlagOutput) != 0) + if ((EmitNodeInfo & OPFL_GlueOutput) != 0) --ResNumResults; // Move the chain reference if needed. if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 && (unsigned)OldChainResultNo != ResNumResults-1) - CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldChainResultNo), + CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldChainResultNo), SDValue(Res, ResNumResults-1)); // Otherwise, no replacement happened because the node already exists. Replace // Uses of the old node with the new one. if (Res != Node) CurDAG->ReplaceAllUsesWith(Node, Res); - + return Res; } -/// CheckPatternPredicate - Implements OP_CheckPatternPredicate. -ALWAYS_INLINE static bool +/// CheckSame - Implements OP_CheckSame. +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SDValue N, const SmallVectorImpl &RecordedNodes) { + SDValue N, + const SmallVectorImpl > &RecordedNodes) { // Accept if it is exactly the same as a previously recorded node. unsigned RecNo = MatcherTable[MatcherIndex++]; assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); - return N == RecordedNodes[RecNo]; + return N == RecordedNodes[RecNo].first; } - + +/// CheckChildSame - Implements OP_CheckChildXSame. +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool +CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, + SDValue N, + const SmallVectorImpl > &RecordedNodes, + unsigned ChildNo) { + if (ChildNo >= N.getNumOperands()) + return false; // Match fails if out of range child #. + return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo), + RecordedNodes); +} + /// CheckPatternPredicate - Implements OP_CheckPatternPredicate. -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SelectionDAGISel &SDISel) { + const SelectionDAGISel &SDISel) { return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]); } /// CheckNodePredicate - Implements OP_CheckNodePredicate. -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SelectionDAGISel &SDISel, SDNode *N) { + const SelectionDAGISel &SDISel, SDNode *N) { return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]); } -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N) { uint16_t Opc = MatcherTable[MatcherIndex++]; @@ -1676,77 +2178,76 @@ CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, return N->getOpcode() == Opc; } -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SDValue N, const TargetLowering &TLI) { + SDValue N, const TargetLowering *TLI) { MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; if (N.getValueType() == VT) return true; - + // Handle the case when VT is iPTR. - return VT == MVT::iPTR && N.getValueType() == TLI.getPointerTy(); + return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(); } -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SDValue N, const TargetLowering &TLI, + SDValue N, const TargetLowering *TLI, unsigned ChildNo) { if (ChildNo >= N.getNumOperands()) return false; // Match fails if out of range child #. return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI); } - -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N) { return cast(N)->get() == (ISD::CondCode)MatcherTable[MatcherIndex++]; } -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SDValue N, const TargetLowering &TLI) { + SDValue N, const TargetLowering *TLI) { MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; if (cast(N)->getVT() == VT) return true; - + // Handle the case when VT is iPTR. - return VT == MVT::iPTR && cast(N)->getVT() == TLI.getPointerTy(); + return VT == MVT::iPTR && cast(N)->getVT() == TLI->getPointerTy(); } -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N) { int64_t Val = MatcherTable[MatcherIndex++]; if (Val & 128) Val = GetVBR(Val, MatcherTable, MatcherIndex); - + ConstantSDNode *C = dyn_cast(N); return C != 0 && C->getSExtValue() == Val; } -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SDValue N, SelectionDAGISel &SDISel) { + SDValue N, const SelectionDAGISel &SDISel) { int64_t Val = MatcherTable[MatcherIndex++]; if (Val & 128) Val = GetVBR(Val, MatcherTable, MatcherIndex); - + if (N->getOpcode() != ISD::AND) return false; - + ConstantSDNode *C = dyn_cast(N->getOperand(1)); return C != 0 && SDISel.CheckAndMask(N.getOperand(0), C, Val); } -ALWAYS_INLINE static bool +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SDValue N, SelectionDAGISel &SDISel) { + SDValue N, const SelectionDAGISel &SDISel) { int64_t Val = MatcherTable[MatcherIndex++]; if (Val & 128) Val = GetVBR(Val, MatcherTable, MatcherIndex); - + if (N->getOpcode() != ISD::OR) return false; - + ConstantSDNode *C = dyn_cast(N->getOperand(1)); return C != 0 && SDISel.CheckOrMask(N.getOperand(0), C, Val); } @@ -1756,11 +2257,12 @@ CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, /// fail, set Result=true and return anything. If the current predicate is /// known to pass, set Result=false and return the MatcherIndex to continue /// with. If the current predicate is unknown, set Result=false and return the -/// MatcherIndex to continue with. +/// MatcherIndex to continue with. static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, - bool &Result, SelectionDAGISel &SDISel, - SmallVectorImpl &RecordedNodes){ + bool &Result, + const SelectionDAGISel &SDISel, + SmallVectorImpl > &RecordedNodes) { switch (Table[Index++]) { default: Result = false; @@ -1768,6 +2270,13 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, case SelectionDAGISel::OPC_CheckSame: Result = !::CheckSame(Table, Index, N, RecordedNodes); return Index; + case SelectionDAGISel::OPC_CheckChild0Same: + case SelectionDAGISel::OPC_CheckChild1Same: + case SelectionDAGISel::OPC_CheckChild2Same: + case SelectionDAGISel::OPC_CheckChild3Same: + Result = !::CheckChildSame(Table, Index, N, RecordedNodes, + Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Same); + return Index; case SelectionDAGISel::OPC_CheckPatternPredicate: Result = !::CheckPatternPredicate(Table, Index, SDISel); return Index; @@ -1778,7 +2287,7 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, Result = !::CheckOpcode(Table, Index, N.getNode()); return Index; case SelectionDAGISel::OPC_CheckType: - Result = !::CheckType(Table, Index, N, SDISel.TLI); + Result = !::CheckType(Table, Index, N, SDISel.getTargetLowering()); return Index; case SelectionDAGISel::OPC_CheckChild0Type: case SelectionDAGISel::OPC_CheckChild1Type: @@ -1788,14 +2297,14 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, case SelectionDAGISel::OPC_CheckChild5Type: case SelectionDAGISel::OPC_CheckChild6Type: case SelectionDAGISel::OPC_CheckChild7Type: - Result = !::CheckChildType(Table, Index, N, SDISel.TLI, + Result = !::CheckChildType(Table, Index, N, SDISel.getTargetLowering(), Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Type); return Index; case SelectionDAGISel::OPC_CheckCondCode: Result = !::CheckCondCode(Table, Index, N); return Index; case SelectionDAGISel::OPC_CheckValueType: - Result = !::CheckValueType(Table, Index, N, SDISel.TLI); + Result = !::CheckValueType(Table, Index, N, SDISel.getTargetLowering()); return Index; case SelectionDAGISel::OPC_CheckInteger: Result = !::CheckInteger(Table, Index, N); @@ -1814,21 +2323,21 @@ namespace { struct MatchScope { /// FailIndex - If this match fails, this is the index to continue with. unsigned FailIndex; - + /// NodeStack - The node stack when the scope was formed. SmallVector NodeStack; - + /// NumRecordedNodes - The number of recorded nodes when the scope was formed. unsigned NumRecordedNodes; - + /// NumMatchedMemRefs - The number of matched memref entries. unsigned NumMatchedMemRefs; - - /// InputChain/InputFlag - The current chain/flag - SDValue InputChain, InputFlag; + + /// InputChain/InputGlue - The current chain/glue + SDValue InputChain, InputGlue; /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty. - bool HasChainNodesMatched, HasFlagResultNodesMatched; + bool HasChainNodesMatched, HasGlueResultNodesMatched; }; } @@ -1843,6 +2352,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, case ISD::EntryToken: // These nodes remain the same. case ISD::BasicBlock: case ISD::Register: + case ISD::RegisterMask: //case ISD::VALUETYPE: //case ISD::CONDCODE: case ISD::HANDLENODE: @@ -1860,6 +2370,8 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, case ISD::CopyFromReg: case ISD::CopyToReg: case ISD::EH_LABEL: + case ISD::LIFETIME_START: + case ISD::LIFETIME_END: NodeToMatch->setNodeId(-1); // Mark selected. return 0; case ISD::AssertSext: @@ -1870,7 +2382,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, case ISD::INLINEASM: return Select_INLINEASM(NodeToMatch); case ISD::UNDEF: return Select_UNDEF(NodeToMatch); } - + assert(!NodeToMatch->isMachineOpcode() && "Node already selected!"); // Set up the node stack with NodeToMatch as the only node on the stack. @@ -1881,42 +2393,43 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, // MatchScopes - Scopes used when matching, if a match failure happens, this // indicates where to continue checking. SmallVector MatchScopes; - + // RecordedNodes - This is the set of nodes that have been recorded by the - // state machine. - SmallVector RecordedNodes; - + // state machine. The second value is the parent of the node, or null if the + // root is recorded. + SmallVector, 8> RecordedNodes; + // MatchedMemRefs - This is the set of MemRef's we've seen in the input // pattern. SmallVector MatchedMemRefs; - - // These are the current input chain and flag for use when generating nodes. + + // These are the current input chain and glue for use when generating nodes. // Various Emit operations change these. For example, emitting a copytoreg // uses and updates these. - SDValue InputChain, InputFlag; - + SDValue InputChain, InputGlue; + // ChainNodesMatched - If a pattern matches nodes that have input/output // chains, the OPC_EmitMergeInputChains operation is emitted which indicates // which ones they are. The result is captured into this list so that we can // update the chain results when the pattern is complete. SmallVector ChainNodesMatched; - SmallVector FlagResultNodesMatched; - - DEBUG(errs() << "ISEL: Starting pattern match on root node: "; + SmallVector GlueResultNodesMatched; + + DEBUG(dbgs() << "ISEL: Starting pattern match on root node: "; NodeToMatch->dump(CurDAG); - errs() << '\n'); - + dbgs() << '\n'); + // Determine where to start the interpreter. Normally we start at opcode #0, // but if the state machine starts with an OPC_SwitchOpcode, then we // accelerate the first lookup (which is guaranteed to be hot) with the // OpcodeOffset table. unsigned MatcherIndex = 0; - + if (!OpcodeOffset.empty()) { // Already computed the OpcodeOffset table, just index into it. if (N.getOpcode() < OpcodeOffset.size()) MatcherIndex = OpcodeOffset[N.getOpcode()]; - DEBUG(errs() << " Initial Opcode index to " << MatcherIndex << "\n"); + DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n"); } else if (MatcherTable[0] == OPC_SwitchOpcode) { // Otherwise, the table isn't computed, but the state machine does start @@ -1943,7 +2456,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, if (N.getOpcode() < OpcodeOffset.size()) MatcherIndex = OpcodeOffset[N.getOpcode()]; } - + while (1) { assert(MatcherIndex < TableSize && "Invalid index"); #ifndef NDEBUG @@ -1958,7 +2471,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, // determine immediately that the first check (or first several) will // immediately fail, don't even bother pushing a scope for them. unsigned FailIndex; - + while (1) { unsigned NumToSkip = MatcherTable[MatcherIndex++]; if (NumToSkip & 128) @@ -1968,12 +2481,12 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, FailIndex = 0; break; } - + FailIndex = MatcherIndex+NumToSkip; - + unsigned MatcherIndexOfPredicate = MatcherIndex; (void)MatcherIndexOfPredicate; // silence warning. - + // If we can't evaluate this predicate without pushing a scope (e.g. if // it is a 'MoveParent') or if the predicate succeeds on this node, we // push the scope and evaluate the full predicate chain. @@ -1982,20 +2495,20 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, Result, *this, RecordedNodes); if (!Result) break; - - DEBUG(errs() << " Skipped scope entry (due to false predicate) at " + + DEBUG(dbgs() << " Skipped scope entry (due to false predicate) at " << "index " << MatcherIndexOfPredicate << ", continuing at " << FailIndex << "\n"); ++NumDAGIselRetries; - + // Otherwise, we know that this case of the Scope is guaranteed to fail, // move to the next case. MatcherIndex = FailIndex; } - + // If the whole scope failed to match, bail. if (FailIndex == 0) break; - + // Push a MatchScope which indicates where to go if the first child fails // to match. MatchScope NewEntry; @@ -2004,17 +2517,21 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, NewEntry.NumRecordedNodes = RecordedNodes.size(); NewEntry.NumMatchedMemRefs = MatchedMemRefs.size(); NewEntry.InputChain = InputChain; - NewEntry.InputFlag = InputFlag; + NewEntry.InputGlue = InputGlue; NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty(); - NewEntry.HasFlagResultNodesMatched = !FlagResultNodesMatched.empty(); + NewEntry.HasGlueResultNodesMatched = !GlueResultNodesMatched.empty(); MatchScopes.push_back(NewEntry); continue; } - case OPC_RecordNode: + case OPC_RecordNode: { // Remember this node, it may end up being an operand in the pattern. - RecordedNodes.push_back(N); + SDNode *Parent = 0; + if (NodeStack.size() > 1) + Parent = NodeStack[NodeStack.size()-2].getNode(); + RecordedNodes.push_back(std::make_pair(N, Parent)); continue; - + } + case OPC_RecordChild0: case OPC_RecordChild1: case OPC_RecordChild2: case OPC_RecordChild3: case OPC_RecordChild4: case OPC_RecordChild5: @@ -2023,20 +2540,21 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, if (ChildNo >= N.getNumOperands()) break; // Match fails if out of range child #. - RecordedNodes.push_back(N->getOperand(ChildNo)); + RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo), + N.getNode())); continue; } case OPC_RecordMemRef: MatchedMemRefs.push_back(cast(N)->getMemOperand()); continue; - - case OPC_CaptureFlagInput: - // If the current node has an input flag, capture it in InputFlag. + + case OPC_CaptureGlueInput: + // If the current node has an input glue, capture it in InputGlue. if (N->getNumOperands() != 0 && - N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Flag) - InputFlag = N->getOperand(N->getNumOperands()-1); + N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue) + InputGlue = N->getOperand(N->getNumOperands()-1); continue; - + case OPC_MoveChild: { unsigned ChildNo = MatcherTable[MatcherIndex++]; if (ChildNo >= N.getNumOperands()) @@ -2045,17 +2563,25 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, NodeStack.push_back(N); continue; } - + case OPC_MoveParent: // Pop the current node off the NodeStack. NodeStack.pop_back(); assert(!NodeStack.empty() && "Node stack imbalance!"); - N = NodeStack.back(); + N = NodeStack.back(); continue; - + case OPC_CheckSame: if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break; continue; + + case OPC_CheckChild0Same: case OPC_CheckChild1Same: + case OPC_CheckChild2Same: case OPC_CheckChild3Same: + if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes, + Opcode-OPC_CheckChild0Same)) + break; + continue; + case OPC_CheckPatternPredicate: if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break; continue; @@ -2068,7 +2594,8 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned CPNum = MatcherTable[MatcherIndex++]; unsigned RecNo = MatcherTable[MatcherIndex++]; assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat"); - if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo], CPNum, + if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second, + RecordedNodes[RecNo].first, CPNum, RecordedNodes)) break; continue; @@ -2076,11 +2603,12 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, case OPC_CheckOpcode: if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break; continue; - + case OPC_CheckType: - if (!::CheckType(MatcherTable, MatcherIndex, N, TLI)) break; + if (!::CheckType(MatcherTable, MatcherIndex, N, getTargetLowering())) + break; continue; - + case OPC_SwitchOpcode: { unsigned CurNodeOpcode = N.getOpcode(); unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart; @@ -2098,22 +2626,22 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, // If the opcode matches, then we will execute this case. if (CurNodeOpcode == Opc) break; - + // Otherwise, skip over this case. MatcherIndex += CaseSize; } - + // If no cases matched, bail out. if (CaseSize == 0) break; - + // Otherwise, execute the case we found. - DEBUG(errs() << " OpcodeSwitch from " << SwitchStart + DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart << " to " << MatcherIndex << "\n"); continue; } - + case OPC_SwitchType: { - MVT::SimpleValueType CurNodeVT = N.getValueType().getSimpleVT().SimpleTy; + MVT CurNodeVT = N.getSimpleValueType(); unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart; unsigned CaseSize; while (1) { @@ -2122,25 +2650,24 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, if (CaseSize & 128) CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex); if (CaseSize == 0) break; - - MVT::SimpleValueType CaseVT = - (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; + + MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; if (CaseVT == MVT::iPTR) - CaseVT = TLI.getPointerTy().SimpleTy; - + CaseVT = getTargetLowering()->getPointerTy(); + // If the VT matches, then we will execute this case. if (CurNodeVT == CaseVT) break; - + // Otherwise, skip over this case. MatcherIndex += CaseSize; } - + // If no cases matched, bail out. if (CaseSize == 0) break; - + // Otherwise, execute the case we found. - DEBUG(errs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString() + DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString() << "] from " << SwitchStart << " to " << MatcherIndex<<'\n'); continue; } @@ -2148,7 +2675,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, case OPC_CheckChild2Type: case OPC_CheckChild3Type: case OPC_CheckChild4Type: case OPC_CheckChild5Type: case OPC_CheckChild6Type: case OPC_CheckChild7Type: - if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI, + if (!::CheckChildType(MatcherTable, MatcherIndex, N, getTargetLowering(), Opcode-OPC_CheckChild0Type)) break; continue; @@ -2156,7 +2683,8 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break; continue; case OPC_CheckValueType: - if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI)) break; + if (!::CheckValueType(MatcherTable, MatcherIndex, N, getTargetLowering())) + break; continue; case OPC_CheckInteger: if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break; @@ -2167,7 +2695,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, case OPC_CheckOrImm: if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break; continue; - + case OPC_CheckFoldableChainNode: { assert(NodeStack.size() != 1 && "No parent node"); // Verify that all intermediate nodes between the root and this one have @@ -2188,7 +2716,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, NodeToMatch, OptLevel, true/*We validate our own chains*/)) break; - + continue; } case OPC_EmitInteger: { @@ -2197,35 +2725,49 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, int64_t Val = MatcherTable[MatcherIndex++]; if (Val & 128) Val = GetVBR(Val, MatcherTable, MatcherIndex); - RecordedNodes.push_back(CurDAG->getTargetConstant(Val, VT)); + RecordedNodes.push_back(std::pair( + CurDAG->getTargetConstant(Val, VT), (SDNode*)0)); continue; } case OPC_EmitRegister: { MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; unsigned RegNo = MatcherTable[MatcherIndex++]; - RecordedNodes.push_back(CurDAG->getRegister(RegNo, VT)); + RecordedNodes.push_back(std::pair( + CurDAG->getRegister(RegNo, VT), (SDNode*)0)); continue; } - + case OPC_EmitRegister2: { + // For targets w/ more than 256 register names, the register enum + // values are stored in two bytes in the matcher table (just like + // opcodes). + MVT::SimpleValueType VT = + (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; + unsigned RegNo = MatcherTable[MatcherIndex++]; + RegNo |= MatcherTable[MatcherIndex++] << 8; + RecordedNodes.push_back(std::pair( + CurDAG->getRegister(RegNo, VT), (SDNode*)0)); + continue; + } + case OPC_EmitConvertToTarget: { // Convert from IMM/FPIMM to target version. unsigned RecNo = MatcherTable[MatcherIndex++]; - assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); - SDValue Imm = RecordedNodes[RecNo]; + assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget"); + SDValue Imm = RecordedNodes[RecNo].first; if (Imm->getOpcode() == ISD::Constant) { - int64_t Val = cast(Imm)->getZExtValue(); - Imm = CurDAG->getTargetConstant(Val, Imm.getValueType()); + const ConstantInt *Val=cast(Imm)->getConstantIntValue(); + Imm = CurDAG->getConstant(*Val, Imm.getValueType(), true); } else if (Imm->getOpcode() == ISD::ConstantFP) { const ConstantFP *Val=cast(Imm)->getConstantFPValue(); - Imm = CurDAG->getTargetConstantFP(*Val, Imm.getValueType()); + Imm = CurDAG->getConstantFP(*Val, Imm.getValueType(), true); } - - RecordedNodes.push_back(Imm); + + RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second)); continue; } - + case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0 case OPC_EmitMergeInputChains1_1: { // OPC_EmitMergeInputChains, 1, 1 // These are space-optimized forms of OPC_EmitMergeInputChains. @@ -2233,28 +2775,28 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, "EmitMergeInputChains should be the first chain producing node"); assert(ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"); - + // Read all of the chained nodes. unsigned RecNo = Opcode == OPC_EmitMergeInputChains1_1; - assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); - ChainNodesMatched.push_back(RecordedNodes[RecNo].getNode()); - + assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains"); + ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode()); + // FIXME: What if other value results of the node have uses not matched // by this pattern? if (ChainNodesMatched.back() != NodeToMatch && - !RecordedNodes[RecNo].hasOneUse()) { + !RecordedNodes[RecNo].first.hasOneUse()) { ChainNodesMatched.clear(); break; } - + // Merge the input chains if they are not intra-pattern references. InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG); - + if (InputChain.getNode() == 0) break; // Failed to merge. continue; } - + case OPC_EmitMergeInputChains: { assert(InputChain.getNode() == 0 && "EmitMergeInputChains should be the first chain producing node"); @@ -2273,55 +2815,56 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, // Read all of the chained nodes. for (unsigned i = 0; i != NumChains; ++i) { unsigned RecNo = MatcherTable[MatcherIndex++]; - assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); - ChainNodesMatched.push_back(RecordedNodes[RecNo].getNode()); - + assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains"); + ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode()); + // FIXME: What if other value results of the node have uses not matched // by this pattern? if (ChainNodesMatched.back() != NodeToMatch && - !RecordedNodes[RecNo].hasOneUse()) { + !RecordedNodes[RecNo].first.hasOneUse()) { ChainNodesMatched.clear(); break; } } - + // If the inner loop broke out, the match fails. if (ChainNodesMatched.empty()) break; // Merge the input chains if they are not intra-pattern references. InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG); - + if (InputChain.getNode() == 0) break; // Failed to merge. continue; } - + case OPC_EmitCopyToReg: { unsigned RecNo = MatcherTable[MatcherIndex++]; - assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); + assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg"); unsigned DestPhysReg = MatcherTable[MatcherIndex++]; - + if (InputChain.getNode() == 0) InputChain = CurDAG->getEntryNode(); - - InputChain = CurDAG->getCopyToReg(InputChain, NodeToMatch->getDebugLoc(), - DestPhysReg, RecordedNodes[RecNo], - InputFlag); - - InputFlag = InputChain.getValue(1); + + InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch), + DestPhysReg, RecordedNodes[RecNo].first, + InputGlue); + + InputGlue = InputChain.getValue(1); continue; } - + case OPC_EmitNodeXForm: { unsigned XFormNo = MatcherTable[MatcherIndex++]; unsigned RecNo = MatcherTable[MatcherIndex++]; - assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); - RecordedNodes.push_back(RunSDNodeXForm(RecordedNodes[RecNo], XFormNo)); + assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm"); + SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo); + RecordedNodes.push_back(std::pair(Res, (SDNode*) 0)); continue; } - + case OPC_EmitNode: case OPC_MorphNodeTo: { uint16_t TargetOpc = MatcherTable[MatcherIndex++]; @@ -2333,15 +2876,15 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, for (unsigned i = 0; i != NumVTs; ++i) { MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; - if (VT == MVT::iPTR) VT = TLI.getPointerTy().SimpleTy; + if (VT == MVT::iPTR) VT = getTargetLowering()->getPointerTy().SimpleTy; VTs.push_back(VT); } - + if (EmitNodeInfo & OPFL_Chain) VTs.push_back(MVT::Other); - if (EmitNodeInfo & OPFL_FlagOutput) - VTs.push_back(MVT::Flag); - + if (EmitNodeInfo & OPFL_GlueOutput) + VTs.push_back(MVT::Glue); + // This is hot code, so optimize the two most common cases of 1 and 2 // results. SDVTList VTList; @@ -2359,11 +2902,11 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned RecNo = MatcherTable[MatcherIndex++]; if (RecNo & 128) RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex); - + assert(RecNo < RecordedNodes.size() && "Invalid EmitNode"); - Ops.push_back(RecordedNodes[RecNo]); + Ops.push_back(RecordedNodes[RecNo].first); } - + // If there are variadic operands to add, handle them now. if (EmitNodeInfo & OPFL_VariadicInfo) { // Determine the start index to copy from. @@ -2371,94 +2914,134 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0; assert(NodeToMatch->getNumOperands() >= FirstOpToCopy && "Invalid variadic node"); - // Copy all of the variadic operands, not including a potential flag + // Copy all of the variadic operands, not including a potential glue // input. for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands(); i != e; ++i) { SDValue V = NodeToMatch->getOperand(i); - if (V.getValueType() == MVT::Flag) break; + if (V.getValueType() == MVT::Glue) break; Ops.push_back(V); } } - - // If this has chain/flag inputs, add them. + + // If this has chain/glue inputs, add them. if (EmitNodeInfo & OPFL_Chain) Ops.push_back(InputChain); - if ((EmitNodeInfo & OPFL_FlagInput) && InputFlag.getNode() != 0) - Ops.push_back(InputFlag); - + if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != 0) + Ops.push_back(InputGlue); + // Create the node. SDNode *Res = 0; if (Opcode != OPC_MorphNodeTo) { // If this is a normal EmitNode command, just create the new node and // add the results to the RecordedNodes list. - Res = CurDAG->getMachineNode(TargetOpc, NodeToMatch->getDebugLoc(), - VTList, Ops.data(), Ops.size()); - - // Add all the non-flag/non-chain results to the RecordedNodes list. + Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch), + VTList, Ops); + + // Add all the non-glue/non-chain results to the RecordedNodes list. for (unsigned i = 0, e = VTs.size(); i != e; ++i) { - if (VTs[i] == MVT::Other || VTs[i] == MVT::Flag) break; - RecordedNodes.push_back(SDValue(Res, i)); + if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break; + RecordedNodes.push_back(std::pair(SDValue(Res, i), + (SDNode*) 0)); } - - } else { + + } else if (NodeToMatch->getOpcode() != ISD::DELETED_NODE) { Res = MorphNode(NodeToMatch, TargetOpc, VTList, Ops.data(), Ops.size(), EmitNodeInfo); + } else { + // NodeToMatch was eliminated by CSE when the target changed the DAG. + // We will visit the equivalent node later. + DEBUG(dbgs() << "Node was eliminated by CSE\n"); + return 0; } - - // If the node had chain/flag results, update our notion of the current - // chain and flag. - if (EmitNodeInfo & OPFL_FlagOutput) { - InputFlag = SDValue(Res, VTs.size()-1); + + // If the node had chain/glue results, update our notion of the current + // chain and glue. + if (EmitNodeInfo & OPFL_GlueOutput) { + InputGlue = SDValue(Res, VTs.size()-1); if (EmitNodeInfo & OPFL_Chain) InputChain = SDValue(Res, VTs.size()-2); } else if (EmitNodeInfo & OPFL_Chain) InputChain = SDValue(Res, VTs.size()-1); - // If the OPFL_MemRefs flag is set on this node, slap all of the + // If the OPFL_MemRefs glue is set on this node, slap all of the // accumulated memrefs onto it. // // FIXME: This is vastly incorrect for patterns with multiple outputs // instructions that access memory and for ComplexPatterns that match // loads. if (EmitNodeInfo & OPFL_MemRefs) { + // Only attach load or store memory operands if the generated + // instruction may load or store. + const MCInstrDesc &MCID = TM.getInstrInfo()->get(TargetOpc); + bool mayLoad = MCID.mayLoad(); + bool mayStore = MCID.mayStore(); + + unsigned NumMemRefs = 0; + for (SmallVectorImpl::const_iterator I = + MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) { + if ((*I)->isLoad()) { + if (mayLoad) + ++NumMemRefs; + } else if ((*I)->isStore()) { + if (mayStore) + ++NumMemRefs; + } else { + ++NumMemRefs; + } + } + MachineSDNode::mmo_iterator MemRefs = - MF->allocateMemRefsArray(MatchedMemRefs.size()); - std::copy(MatchedMemRefs.begin(), MatchedMemRefs.end(), MemRefs); + MF->allocateMemRefsArray(NumMemRefs); + + MachineSDNode::mmo_iterator MemRefsPos = MemRefs; + for (SmallVectorImpl::const_iterator I = + MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) { + if ((*I)->isLoad()) { + if (mayLoad) + *MemRefsPos++ = *I; + } else if ((*I)->isStore()) { + if (mayStore) + *MemRefsPos++ = *I; + } else { + *MemRefsPos++ = *I; + } + } + cast(Res) - ->setMemRefs(MemRefs, MemRefs + MatchedMemRefs.size()); + ->setMemRefs(MemRefs, MemRefs + NumMemRefs); } - - DEBUG(errs() << " " + + DEBUG(dbgs() << " " << (Opcode == OPC_MorphNodeTo ? "Morphed" : "Created") - << " node: "; Res->dump(CurDAG); errs() << "\n"); - + << " node: "; Res->dump(CurDAG); dbgs() << "\n"); + // If this was a MorphNodeTo then we're completely done! if (Opcode == OPC_MorphNodeTo) { - // Update chain and flag uses. - UpdateChainsAndFlags(NodeToMatch, InputChain, ChainNodesMatched, - InputFlag, FlagResultNodesMatched, true); + // Update chain and glue uses. + UpdateChainsAndGlue(NodeToMatch, InputChain, ChainNodesMatched, + InputGlue, GlueResultNodesMatched, true); return Res; } - + continue; } - - case OPC_MarkFlagResults: { + + case OPC_MarkGlueResults: { unsigned NumNodes = MatcherTable[MatcherIndex++]; - - // Read and remember all the flag-result nodes. + + // Read and remember all the glue-result nodes. for (unsigned i = 0; i != NumNodes; ++i) { unsigned RecNo = MatcherTable[MatcherIndex++]; if (RecNo & 128) RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex); - assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); - FlagResultNodesMatched.push_back(RecordedNodes[RecNo].getNode()); + assert(RecNo < RecordedNodes.size() && "Invalid MarkGlueResults"); + GlueResultNodesMatched.push_back(RecordedNodes[RecNo].first.getNode()); } continue; } - + case OPC_CompleteMatch: { // The match has been completed, and any new nodes (if any) have been // created. Patch up references to the matched dag to use the newly @@ -2469,13 +3052,13 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned ResSlot = MatcherTable[MatcherIndex++]; if (ResSlot & 128) ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex); - - assert(ResSlot < RecordedNodes.size() && "Invalid CheckSame"); - SDValue Res = RecordedNodes[ResSlot]; - + + assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch"); + SDValue Res = RecordedNodes[ResSlot].first; + assert(i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && - NodeToMatch->getValueType(i) != MVT::Flag && + NodeToMatch->getValueType(i) != MVT::Glue && "Invalid number of results to complete!"); assert((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || @@ -2486,28 +3069,27 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, i), Res); } - // If the root node defines a flag, add it to the flag nodes to update - // list. - if (NodeToMatch->getValueType(NodeToMatch->getNumValues()-1) == MVT::Flag) - FlagResultNodesMatched.push_back(NodeToMatch); - - // Update chain and flag uses. - UpdateChainsAndFlags(NodeToMatch, InputChain, ChainNodesMatched, - InputFlag, FlagResultNodesMatched, false); - + // If the root node defines glue, add it to the glue nodes to update list. + if (NodeToMatch->getValueType(NodeToMatch->getNumValues()-1) == MVT::Glue) + GlueResultNodesMatched.push_back(NodeToMatch); + + // Update chain and glue uses. + UpdateChainsAndGlue(NodeToMatch, InputChain, ChainNodesMatched, + InputGlue, GlueResultNodesMatched, false); + assert(NodeToMatch->use_empty() && "Didn't replace all uses of the node?"); - + // FIXME: We just return here, which interacts correctly with SelectRoot // above. We should fix this to not return an SDNode* anymore. return 0; } } - + // If the code reached this point, then the match failed. See if there is // another child to try in the current 'Scope', otherwise pop it until we // find a case to check. - DEBUG(errs() << " Match failed at index " << CurrentOpcodeIndex << "\n"); + DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex << "\n"); ++NumDAGIselRetries; while (1) { if (MatchScopes.empty()) { @@ -2526,15 +3108,15 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size()) MatchedMemRefs.resize(LastScope.NumMatchedMemRefs); MatcherIndex = LastScope.FailIndex; - - DEBUG(errs() << " Continuing at " << MatcherIndex << "\n"); - + + DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n"); + InputChain = LastScope.InputChain; - InputFlag = LastScope.InputFlag; + InputGlue = LastScope.InputGlue; if (!LastScope.HasChainNodesMatched) ChainNodesMatched.clear(); - if (!LastScope.HasFlagResultNodesMatched) - FlagResultNodesMatched.clear(); + if (!LastScope.HasGlueResultNodesMatched) + GlueResultNodesMatched.clear(); // Check to see what the offset is at the new MatcherIndex. If it is zero // we have reached the end of this scope, otherwise we have another child @@ -2549,25 +3131,26 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, LastScope.FailIndex = MatcherIndex+NumToSkip; break; } - + // End of this scope, pop it and try the next child in the containing // scope. MatchScopes.pop_back(); } } } - + void SelectionDAGISel::CannotYetSelect(SDNode *N) { std::string msg; raw_string_ostream Msg(msg); - Msg << "Cannot yet select: "; - + Msg << "Cannot select: "; + if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN && N->getOpcode() != ISD::INTRINSIC_WO_CHAIN && N->getOpcode() != ISD::INTRINSIC_VOID) { N->printrFull(Msg, CurDAG); + Msg << "\nIn function: " << MF->getName(); } else { bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other; unsigned iid =