X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAGISel.cpp;h=4b88f7884f5d8287e2c303520b1d92835395c8e6;hb=5bab78527a946632cd9614daa0b9a82ee7b5e1cc;hp=3312542d390f4a8b1542efa6550d1d69ab9ccaa3;hpb=f664e41b201bad27ed3661bf50cd71f54242c114;p=oota-llvm.git

diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 3312542d390..4b88f7884f5 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2,8 +2,8 @@
 //
 //                     The LLVM Compiler Infrastructure
 //
-// This file was developed by the LLVM research group and is distributed under
-// the University of Illinois Open Source License. See LICENSE.TXT for details.
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
@@ -26,14 +26,15 @@
 #include "llvm/Intrinsics.h"
 #include "llvm/IntrinsicInst.h"
 #include "llvm/ParameterAttributes.h"
-#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Collector.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SchedulerRegistry.h"
 #include "llvm/CodeGen/SelectionDAG.h"
-#include "llvm/CodeGen/SSARegMap.h"
 #include "llvm/Target/MRegisterInfo.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetFrameInfo.h"
@@ -54,8 +55,11 @@ ViewISelDAGs("view-isel-dags", cl::Hidden,
 static cl::opt<bool>
 ViewSchedDAGs("view-sched-dags", cl::Hidden,
           cl::desc("Pop up a window to show sched dags as they are processed"));
+static cl::opt<bool>
+ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
+      cl::desc("Pop up a window to show SUnit dags after they are processed"));
 #else
-static const bool ViewISelDAGs = 0, ViewSchedDAGs = 0;
+static const bool ViewISelDAGs = 0, ViewSchedDAGs = 0, ViewSUnitDAGs = 0;
 #endif
 
 //===---------------------------------------------------------------------===//
@@ -73,9 +77,10 @@ MachinePassRegistry RegisterScheduler::Registry;
 namespace {
   cl::opt<RegisterScheduler::FunctionPassCtor, false,
           RegisterPassParser<RegisterScheduler> >
-  ISHeuristic("sched",
+  ISHeuristic("pre-RA-sched",
               cl::init(&createDefaultScheduler),
-              cl::desc("Instruction schedulers available:"));
+              cl::desc("Instruction schedulers available (before register"
+                       " allocation):"));
 
   static RegisterScheduler
   defaultListDAGScheduler("default", "  Best scheduler for the target",
@@ -163,7 +168,7 @@ namespace llvm {
     TargetLowering &TLI;
     Function &Fn;
     MachineFunction &MF;
-    SSARegMap *RegMap;
+    MachineRegisterInfo &RegInfo;
 
     FunctionLoweringInfo(TargetLowering &TLI, Function &Fn,MachineFunction &MF);
 
@@ -186,7 +191,7 @@ namespace llvm {
 #endif
 
     unsigned MakeReg(MVT::ValueType VT) {
-      return RegMap->createVirtualRegister(TLI.getRegClassFor(VT));
+      return RegInfo.createVirtualRegister(TLI.getRegClassFor(VT));
     }
     
     /// isExportedInst - Return true if the specified value is an instruction
@@ -209,7 +214,8 @@ namespace llvm {
 /// eh.selector intrinsic.
 static bool isSelector(Instruction *I) {
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
-    return II->getIntrinsicID() == Intrinsic::eh_selector;
+    return (II->getIntrinsicID() == Intrinsic::eh_selector_i32 ||
+            II->getIntrinsicID() == Intrinsic::eh_selector_i64);
   return false;
 }
 
@@ -240,7 +246,7 @@ static bool isOnlyUsedInEntryBlock(Argument *A) {
 
 FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli,
                                            Function &fn, MachineFunction &mf)
-    : TLI(tli), Fn(fn), MF(mf), RegMap(MF.getSSARegMap()) {
+    : TLI(tli), Fn(fn), MF(mf), RegInfo(MF.getRegInfo()) {
 
   // Create a vreg for each argument register that is not dead and is used
   // outside of the entry block for the function.
@@ -257,7 +263,7 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli,
     if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
       if (ConstantInt *CUI = dyn_cast<ConstantInt>(AI->getArraySize())) {
         const Type *Ty = AI->getAllocatedType();
-        uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+        uint64_t TySize = TLI.getTargetData()->getABITypeSize(Ty);
         unsigned Align = 
           std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty),
                    AI->getAlignment());
@@ -409,6 +415,7 @@ public:
   TargetLowering &TLI;
   SelectionDAG &DAG;
   const TargetData *TD;
+  AliasAnalysis &AA;
 
   /// SwitchCases - Vector of CaseBlock structures used to communicate
   /// SwitchInst code generation information.
@@ -421,11 +428,16 @@ public:
   /// FuncInfo - Information about the function as a whole.
   ///
   FunctionLoweringInfo &FuncInfo;
+  
+  /// GCI - Garbage collection metadata for the function.
+  CollectorMetadata *GCI;
 
   SelectionDAGLowering(SelectionDAG &dag, TargetLowering &tli,
-                       FunctionLoweringInfo &funcinfo)
-    : TLI(tli), DAG(dag), TD(DAG.getTarget().getTargetData()),
-      FuncInfo(funcinfo) {
+                       AliasAnalysis &aa,
+                       FunctionLoweringInfo &funcinfo,
+                       CollectorMetadata *gci)
+    : TLI(tli), DAG(dag), TD(DAG.getTarget().getTargetData()), AA(aa),
+      FuncInfo(funcinfo), GCI(gci) {
   }
 
   /// getRoot - Return the current virtual root of the Selection DAG.
@@ -472,10 +484,6 @@ public:
                         const Value *SV, SDOperand Root,
                         bool isVolatile, unsigned Alignment);
 
-  SDOperand getIntPtrConstant(uint64_t Val) {
-    return DAG.getConstant(Val, TLI.getPointerTy());
-  }
-
   SDOperand getValue(const Value *V);
 
   void setValue(const Value *V, SDOperand NewN) {
@@ -493,11 +501,9 @@ public:
                             unsigned Opc);
   bool isExportableFromCurrentBlock(Value *V, const BasicBlock *FromBB);
   void ExportFromCurrentBlock(Value *V);
-  void LowerCallTo(Instruction &I,
-                   const Type *CalledValueTy, unsigned CallingConv,
-                   bool IsTailCall, SDOperand Callee, unsigned OpIdx,
+  void LowerCallTo(CallSite CS, SDOperand Callee, bool IsTailCall,
                    MachineBasicBlock *LandingPad = NULL);
-  
+
   // Terminator instructions.
   void visitRet(ReturnInst &I);
   void visitBr(BranchInst &I);
@@ -591,7 +597,7 @@ public:
   void visitStore(StoreInst &I);
   void visitPHI(PHINode &I) { } // PHI nodes are handled specially.
   void visitCall(CallInst &I);
-  void visitInlineAsm(CallInst &I);
+  void visitInlineAsm(CallSite CS);
   const char *visitIntrinsicCall(CallInst &I, unsigned Intrinsic);
   void visitTargetIntrinsic(CallInst &I, unsigned Intrinsic);
 
@@ -621,7 +627,6 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG,
                                   unsigned NumParts,
                                   MVT::ValueType PartVT,
                                   MVT::ValueType ValueVT,
-                                  bool EndianOrder,
                                   ISD::NodeType AssertOp = ISD::DELETED_NODE) {
   if (!MVT::isVector(ValueVT) || NumParts == 1) {
     SDOperand Val = Parts[0];
@@ -631,7 +636,7 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG,
       assert(NumParts == 2 &&
              "Cannot expand to more than 2 elts yet!");
       SDOperand Hi = Parts[1];
-      if (EndianOrder && !DAG.getTargetLoweringInfo().isLittleEndian())
+      if (!DAG.getTargetLoweringInfo().isLittleEndian())
         std::swap(Val, Hi);
       return DAG.getNode(ISD::BUILD_PAIR, ValueVT, Val, Hi);
     }
@@ -643,7 +648,15 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG,
   
     if (MVT::isVector(PartVT)) {
       assert(MVT::isVector(ValueVT) && "Unknown vector conversion!");
-      return DAG.getNode(ISD::BIT_CONVERT, PartVT, Val);
+      return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val);
+    }
+  
+    if (MVT::isVector(ValueVT)) {
+      assert(NumParts == 1 &&
+             MVT::getVectorElementType(ValueVT) == PartVT &&
+             MVT::getVectorNumElements(ValueVT) == 1 &&
+             "Only trivial scalar-to-vector conversions should get here!");
+      return DAG.getNode(ISD::BUILD_VECTOR, ValueVT, Val);
     }
   
     if (MVT::isInteger(PartVT) &&
@@ -661,12 +674,10 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG,
       }
     }
   
-    if (MVT::isFloatingPoint(PartVT) &&
-        MVT::isFloatingPoint(ValueVT))
-      return DAG.getNode(ISD::FP_ROUND, ValueVT, Val);
+    if (MVT::isFloatingPoint(PartVT) && MVT::isFloatingPoint(ValueVT))
+      return DAG.getNode(ISD::FP_ROUND, ValueVT, Val, DAG.getIntPtrConstant(0));
 
-    if (MVT::getSizeInBits(PartVT) == 
-        MVT::getSizeInBits(ValueVT))
+    if (MVT::getSizeInBits(PartVT) == MVT::getSizeInBits(ValueVT))
       return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val);
 
     assert(0 && "Unknown mismatch!");
@@ -692,16 +703,16 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG,
     // as appropriate.
     for (unsigned i = 0; i != NumParts; ++i)
       Ops[i] = getCopyFromParts(DAG, &Parts[i], 1,
-                                PartVT, IntermediateVT, EndianOrder);
+                                PartVT, IntermediateVT);
   } else if (NumParts > 0) {
     // If the intermediate type was expanded, build the intermediate operands
     // from the parts.
-    assert(NumIntermediates % NumParts == 0 &&
+    assert(NumParts % NumIntermediates == 0 &&
            "Must expand into a divisible number of parts!");
-    unsigned Factor = NumIntermediates / NumParts;
+    unsigned Factor = NumParts / NumIntermediates;
     for (unsigned i = 0; i != NumIntermediates; ++i)
       Ops[i] = getCopyFromParts(DAG, &Parts[i * Factor], Factor,
-                                PartVT, IntermediateVT, EndianOrder);
+                                PartVT, IntermediateVT);
   }
   
   // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the intermediate
@@ -709,7 +720,7 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG,
   return DAG.getNode(MVT::isVector(IntermediateVT) ?
                        ISD::CONCAT_VECTORS :
                        ISD::BUILD_VECTOR,
-                     ValueVT, &Ops[0], NumParts);
+                     ValueVT, &Ops[0], NumIntermediates);
 }
 
 /// getCopyToParts - Create a series of nodes that contain the
@@ -718,8 +729,9 @@ static void getCopyToParts(SelectionDAG &DAG,
                            SDOperand Val,
                            SDOperand *Parts,
                            unsigned NumParts,
-                           MVT::ValueType PartVT,
-                           bool EndianOrder) {
+                           MVT::ValueType PartVT) {
+  TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  MVT::ValueType PtrVT = TLI.getPointerTy();
   MVT::ValueType ValueVT = Val.getValueType();
 
   if (!MVT::isVector(ValueVT) || NumParts == 1) {
@@ -727,8 +739,8 @@ static void getCopyToParts(SelectionDAG &DAG,
     if (NumParts > 1) {
       for (unsigned i = 0; i != NumParts; ++i)
         Parts[i] = DAG.getNode(ISD::EXTRACT_ELEMENT, PartVT, Val,
-                               DAG.getConstant(i, MVT::i32));
-      if (EndianOrder && !DAG.getTargetLoweringInfo().isLittleEndian())
+                               DAG.getConstant(i, PtrVT));
+      if (!DAG.getTargetLoweringInfo().isLittleEndian())
         std::reverse(Parts, Parts + NumParts);
       return;
     }
@@ -740,6 +752,13 @@ static void getCopyToParts(SelectionDAG &DAG,
         assert(MVT::isVector(ValueVT) &&
                "Not a vector-vector cast?");
         Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val);
+      } else if (MVT::isVector(ValueVT)) {
+        assert(NumParts == 1 &&
+               MVT::getVectorElementType(ValueVT) == PartVT &&
+               MVT::getVectorNumElements(ValueVT) == 1 &&
+               "Only trivial vector-to-scalar conversions should get here!");
+        Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, PartVT, Val,
+                          DAG.getConstant(0, PtrVT));
       } else if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) {
         if (PartVT < ValueVT)
           Val = DAG.getNode(ISD::TRUNCATE, PartVT, Val);
@@ -778,18 +797,18 @@ static void getCopyToParts(SelectionDAG &DAG,
       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR,
                            IntermediateVT, Val,
                            DAG.getConstant(i * (NumElements / NumIntermediates),
-                                           MVT::i32));
+                                           PtrVT));
     else
       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
                            IntermediateVT, Val, 
-                           DAG.getConstant(i, MVT::i32));
+                           DAG.getConstant(i, PtrVT));
 
   // Split the intermediate operands into legal parts.
   if (NumParts == NumIntermediates) {
     // If the register was not expanded, promote or copy the value,
     // as appropriate.
     for (unsigned i = 0; i != NumParts; ++i)
-      getCopyToParts(DAG, Ops[i], &Parts[i], 1, PartVT, EndianOrder);
+      getCopyToParts(DAG, Ops[i], &Parts[i], 1, PartVT);
   } else if (NumParts > 0) {
     // If the intermediate type was expanded, split each the value into
     // legal parts.
@@ -797,7 +816,7 @@ static void getCopyToParts(SelectionDAG &DAG,
            "Must expand into a divisible number of parts!");
     unsigned Factor = NumParts / NumIntermediates;
     for (unsigned i = 0; i != NumIntermediates; ++i)
-      getCopyToParts(DAG, Ops[i], &Parts[i * Factor], Factor, PartVT, EndianOrder);
+      getCopyToParts(DAG, Ops[i], &Parts[i * Factor], Factor, PartVT);
   }
 }
 
@@ -835,20 +854,20 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) {
       return N = DAG.getNode(ISD::BUILD_VECTOR, VT,
                              &Ops[0], Ops.size());
     } else if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
-      return N = DAG.getConstantFP(CFP->getValue(), VT);
+      return N = DAG.getConstantFP(CFP->getValueAPF(), VT);
     } else if (const VectorType *PTy = dyn_cast<VectorType>(VTy)) {
       unsigned NumElements = PTy->getNumElements();
       MVT::ValueType PVT = TLI.getValueType(PTy->getElementType());
       
       // Now that we know the number and type of the elements, push a
       // Constant or ConstantFP node onto the ops list for each element of
-      // the packed constant.
+      // the vector constant.
       SmallVector<SDOperand, 8> Ops;
       if (ConstantVector *CP = dyn_cast<ConstantVector>(C)) {
         for (unsigned i = 0; i != NumElements; ++i)
           Ops.push_back(getValue(CP->getOperand(i)));
       } else {
-        assert(isa<ConstantAggregateZero>(C) && "Unknown packed constant!");
+        assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
         SDOperand Op;
         if (MVT::isFloatingPoint(PVT))
           Op = DAG.getConstantFP(0, PVT);
@@ -913,12 +932,11 @@ void SelectionDAGLowering::visitRet(ReturnInst &I) {
         TmpVT = TLI.getTypeToTransformTo(MVT::i32);
       else
         TmpVT = MVT::i32;
-      const FunctionType *FTy = I.getParent()->getParent()->getFunctionType();
-      const ParamAttrsList *Attrs = FTy->getParamAttrs();
+      const Function *F = I.getParent()->getParent();
       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
-      if (Attrs && Attrs->paramHasAttr(0, ParamAttr::SExt))
+      if (F->paramHasAttr(0, ParamAttr::SExt))
         ExtendKind = ISD::SIGN_EXTEND;
-      if (Attrs && Attrs->paramHasAttr(0, ParamAttr::ZExt))
+      if (F->paramHasAttr(0, ParamAttr::ZExt))
         ExtendKind = ISD::ZERO_EXTEND;
       RetOp = DAG.getNode(ExtendKind, TmpVT, RetOp);
       NewValues.push_back(RetOp);
@@ -928,7 +946,7 @@ void SelectionDAGLowering::visitRet(ReturnInst &I) {
       unsigned NumParts = TLI.getNumRegisters(VT);
       MVT::ValueType PartVT = TLI.getRegisterType(VT);
       SmallVector<SDOperand, 4> Parts(NumParts);
-      getCopyToParts(DAG, RetOp, &Parts[0], NumParts, PartVT, true);
+      getCopyToParts(DAG, RetOp, &Parts[0], NumParts, PartVT);
       for (unsigned i = 0; i < NumParts; ++i) {
         NewValues.push_back(Parts[i]);
         NewValues.push_back(DAG.getConstant(false, MVT::i32));
@@ -1146,7 +1164,6 @@ void SelectionDAGLowering::visitBr(BranchInst &I) {
 
     // Update machine-CFG edges.
     CurMBB->addSuccessor(Succ0MBB);
-
     return;
   }
 
@@ -1429,11 +1446,10 @@ void SelectionDAGLowering::visitInvoke(InvokeInst &I) {
   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
 
-  LowerCallTo(I, I.getCalledValue()->getType(),
-              I.getCallingConv(),
-              false,
-              getValue(I.getOperand(0)),
-              3, LandingPad);
+  if (isa<InlineAsm>(I.getCalledValue()))
+    visitInlineAsm(&I);
+  else
+    LowerCallTo(&I, getValue(I.getOperand(0)), false, LandingPad);
 
   // If the value of the invoke is used outside of its defining block, make it
   // available as a virtual register.
@@ -1894,7 +1910,7 @@ unsigned SelectionDAGLowering::Clusterify(CaseVector& Cases,
                          SI.getSuccessorValue(i),
                          SMBB));
   }
-  sort(Cases.begin(), Cases.end(), CaseCmp());
+  std::sort(Cases.begin(), Cases.end(), CaseCmp());
 
   // Merge case into clusters
   if (Cases.size()>=2)
@@ -1998,7 +2014,7 @@ void SelectionDAGLowering::visitSub(User &I) {
       const Type *ElTy = DestTy->getElementType();
       if (ElTy->isFloatingPoint()) {
         unsigned VL = DestTy->getNumElements();
-        std::vector<Constant*> NZ(VL, ConstantFP::get(ElTy, -0.0));
+        std::vector<Constant*> NZ(VL, ConstantFP::getNegativeZero(ElTy));
         Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size());
         if (CV == CNZ) {
           SDOperand Op2 = getValue(I.getOperand(1));
@@ -2010,7 +2026,7 @@ void SelectionDAGLowering::visitSub(User &I) {
   }
   if (Ty->isFloatingPoint()) {
     if (ConstantFP *CFP = dyn_cast<ConstantFP>(I.getOperand(0)))
-      if (CFP->isExactlyValue(-0.0)) {
+      if (CFP->isExactlyValue(ConstantFP::getNegativeZero(Ty)->getValueAPF())) {
         SDOperand Op2 = getValue(I.getOperand(1));
         setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2));
         return;
@@ -2142,7 +2158,7 @@ void SelectionDAGLowering::visitFPTrunc(User &I) {
   // FPTrunc is never a no-op cast, no need to check
   SDOperand N = getValue(I.getOperand(0));
   MVT::ValueType DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N));
+  setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N, DAG.getIntPtrConstant(0)));
 }
 
 void SelectionDAGLowering::visitFPExt(User &I){ 
@@ -2263,7 +2279,7 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) {
         // N = N + Offset
         uint64_t Offset = TD->getStructLayout(StTy)->getElementOffset(Field);
         N = DAG.getNode(ISD::ADD, N.getValueType(), N,
-                        getIntPtrConstant(Offset));
+                        DAG.getIntPtrConstant(Offset));
       }
       Ty = StTy->getElementType(Field);
     } else {
@@ -2273,13 +2289,14 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) {
       if (ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
         if (CI->getZExtValue() == 0) continue;
         uint64_t Offs = 
-            TD->getTypeSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
-        N = DAG.getNode(ISD::ADD, N.getValueType(), N, getIntPtrConstant(Offs));
+            TD->getABITypeSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
+        N = DAG.getNode(ISD::ADD, N.getValueType(), N,
+                        DAG.getIntPtrConstant(Offs));
         continue;
       }
       
       // N = N + Idx * ElementSize;
-      uint64_t ElementSize = TD->getTypeSize(Ty);
+      uint64_t ElementSize = TD->getABITypeSize(Ty);
       SDOperand IdxN = getValue(Idx);
 
       // If the index is smaller or larger than intptr_t, truncate or extend
@@ -2299,7 +2316,7 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) {
         continue;
       }
       
-      SDOperand Scale = getIntPtrConstant(ElementSize);
+      SDOperand Scale = DAG.getIntPtrConstant(ElementSize);
       IdxN = DAG.getNode(ISD::MUL, N.getValueType(), IdxN, Scale);
       N = DAG.getNode(ISD::ADD, N.getValueType(), N, IdxN);
     }
@@ -2314,7 +2331,7 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) {
     return;   // getValue will auto-populate this.
 
   const Type *Ty = I.getAllocatedType();
-  uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+  uint64_t TySize = TLI.getTargetData()->getABITypeSize(Ty);
   unsigned Align =
     std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty),
              I.getAlignment());
@@ -2327,25 +2344,25 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) {
     AllocSize = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, AllocSize);
 
   AllocSize = DAG.getNode(ISD::MUL, IntPtr, AllocSize,
-                          getIntPtrConstant(TySize));
+                          DAG.getIntPtrConstant(TySize));
 
-  // Handle alignment.  If the requested alignment is less than or equal to the
-  // stack alignment, ignore it and round the size of the allocation up to the
-  // stack alignment size.  If the size is greater than the stack alignment, we
-  // note this in the DYNAMIC_STACKALLOC node.
+  // Handle alignment.  If the requested alignment is less than or equal to
+  // the stack alignment, ignore it.  If the size is greater than or equal to
+  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
   unsigned StackAlign =
     TLI.getTargetMachine().getFrameInfo()->getStackAlignment();
-  if (Align <= StackAlign) {
+  if (Align <= StackAlign)
     Align = 0;
-    // Add SA-1 to the size.
-    AllocSize = DAG.getNode(ISD::ADD, AllocSize.getValueType(), AllocSize,
-                            getIntPtrConstant(StackAlign-1));
-    // Mask out the low bits for alignment purposes.
-    AllocSize = DAG.getNode(ISD::AND, AllocSize.getValueType(), AllocSize,
-                            getIntPtrConstant(~(uint64_t)(StackAlign-1)));
-  }
 
-  SDOperand Ops[] = { getRoot(), AllocSize, getIntPtrConstant(Align) };
+  // Round the size of the allocation up to the stack alignment size
+  // by add SA-1 to the size.
+  AllocSize = DAG.getNode(ISD::ADD, AllocSize.getValueType(), AllocSize,
+                          DAG.getIntPtrConstant(StackAlign-1));
+  // Mask out the low bits for alignment purposes.
+  AllocSize = DAG.getNode(ISD::AND, AllocSize.getValueType(), AllocSize,
+                          DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
+
+  SDOperand Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
   const MVT::ValueType *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(),
                                                     MVT::Other);
   SDOperand DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, 2, Ops, 3);
@@ -2397,31 +2414,13 @@ void SelectionDAGLowering::visitStore(StoreInst &I) {
                            I.isVolatile(), I.getAlignment()));
 }
 
-/// IntrinsicCannotAccessMemory - Return true if the specified intrinsic cannot
-/// access memory and has no other side effects at all.
-static bool IntrinsicCannotAccessMemory(unsigned IntrinsicID) {
-#define GET_NO_MEMORY_INTRINSICS
-#include "llvm/Intrinsics.gen"
-#undef GET_NO_MEMORY_INTRINSICS
-  return false;
-}
-
-// IntrinsicOnlyReadsMemory - Return true if the specified intrinsic doesn't
-// have any side-effects or if it only reads memory.
-static bool IntrinsicOnlyReadsMemory(unsigned IntrinsicID) {
-#define GET_SIDE_EFFECT_INFO
-#include "llvm/Intrinsics.gen"
-#undef GET_SIDE_EFFECT_INFO
-  return false;
-}
-
 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
 /// node.
 void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, 
                                                 unsigned Intrinsic) {
-  bool HasChain = !IntrinsicCannotAccessMemory(Intrinsic);
-  bool OnlyLoad = HasChain && IntrinsicOnlyReadsMemory(Intrinsic);
-  
+  bool HasChain = !I.doesNotAccessMemory();
+  bool OnlyLoad = HasChain && I.onlyReadsMemory();
+
   // Build the operand list.
   SmallVector<SDOperand, 8> Ops;
   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
@@ -2519,7 +2518,7 @@ static void addCatchInfo(CallInst &I, MachineModuleInfo *MMI,
   for (unsigned i = N - 1; i > 2; --i) {
     if (ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(i))) {
       unsigned FilterLength = CI->getZExtValue();
-      unsigned FirstCatch = i + FilterLength + 1;
+      unsigned FirstCatch = i + FilterLength + !FilterLength;
       assert (FirstCatch <= N && "Invalid filter length");
 
       if (FirstCatch < N) {
@@ -2530,11 +2529,17 @@ static void addCatchInfo(CallInst &I, MachineModuleInfo *MMI,
         TyInfo.clear();
       }
 
-      TyInfo.reserve(FilterLength);
-      for (unsigned j = i + 1; j < FirstCatch; ++j)
-        TyInfo.push_back(ExtractTypeInfo(I.getOperand(j)));
-      MMI->addFilterTypeInfo(MBB, TyInfo);
-      TyInfo.clear();
+      if (!FilterLength) {
+        // Cleanup.
+        MMI->addCleanup(MBB);
+      } else {
+        // Filter.
+        TyInfo.reserve(FilterLength - 1);
+        for (unsigned j = i + 1; j < FirstCatch; ++j)
+          TyInfo.push_back(ExtractTypeInfo(I.getOperand(j)));
+        MMI->addFilterTypeInfo(MBB, TyInfo);
+        TyInfo.clear();
+      }
 
       N = i;
     }
@@ -2676,9 +2681,12 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     return 0;
   }
 
-  case Intrinsic::eh_selector:{
+  case Intrinsic::eh_selector_i32:
+  case Intrinsic::eh_selector_i64: {
     MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
-
+    MVT::ValueType VT = (Intrinsic == Intrinsic::eh_selector_i32 ?
+                         MVT::i32 : MVT::i64);
+    
     if (ExceptionHandling && MMI) {
       if (CurMBB->isLandingPad())
         addCatchInfo(I, MMI, CurMBB);
@@ -2692,7 +2700,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
       }
 
       // Insert the EHSELECTION instruction.
-      SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other);
+      SDVTList VTs = DAG.getVTList(VT, MVT::Other);
       SDOperand Ops[2];
       Ops[0] = getValue(I.getOperand(1));
       Ops[1] = getRoot();
@@ -2700,42 +2708,114 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
       setValue(&I, Op);
       DAG.setRoot(Op.getValue(1));
     } else {
-      setValue(&I, DAG.getConstant(0, TLI.getPointerTy()));
+      setValue(&I, DAG.getConstant(0, VT));
     }
     
     return 0;
   }
-  
-  case Intrinsic::eh_typeid_for: {
+
+  case Intrinsic::eh_typeid_for_i32:
+  case Intrinsic::eh_typeid_for_i64: {
     MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+    MVT::ValueType VT = (Intrinsic == Intrinsic::eh_typeid_for_i32 ?
+                         MVT::i32 : MVT::i64);
     
     if (MMI) {
       // Find the type id for the given typeinfo.
       GlobalVariable *GV = ExtractTypeInfo(I.getOperand(1));
 
       unsigned TypeID = MMI->getTypeIDFor(GV);
-      setValue(&I, DAG.getConstant(TypeID, MVT::i32));
+      setValue(&I, DAG.getConstant(TypeID, VT));
     } else {
       // Return something different to eh_selector.
-      setValue(&I, DAG.getConstant(1, MVT::i32));
+      setValue(&I, DAG.getConstant(1, VT));
+    }
+
+    return 0;
+  }
+
+  case Intrinsic::eh_return: {
+    MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+
+    if (MMI && ExceptionHandling) {
+      MMI->setCallsEHReturn(true);
+      DAG.setRoot(DAG.getNode(ISD::EH_RETURN,
+                              MVT::Other,
+                              getRoot(),
+                              getValue(I.getOperand(1)),
+                              getValue(I.getOperand(2))));
+    } else {
+      setValue(&I, DAG.getConstant(0, TLI.getPointerTy()));
     }
 
     return 0;
   }
 
-  case Intrinsic::sqrt_f32:
-  case Intrinsic::sqrt_f64:
+   case Intrinsic::eh_unwind_init: {    
+     if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) {
+       MMI->setCallsUnwindInit(true);
+     }
+
+     return 0;
+   }
+
+   case Intrinsic::eh_dwarf_cfa: {
+     if (ExceptionHandling) {
+       MVT::ValueType VT = getValue(I.getOperand(1)).getValueType();
+       SDOperand CfaArg;
+       if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy()))
+         CfaArg = DAG.getNode(ISD::TRUNCATE,
+                              TLI.getPointerTy(), getValue(I.getOperand(1)));
+       else
+         CfaArg = DAG.getNode(ISD::SIGN_EXTEND,
+                              TLI.getPointerTy(), getValue(I.getOperand(1)));
+       
+       SDOperand Offset = DAG.getNode(ISD::ADD,
+                                      TLI.getPointerTy(),
+                                      DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET,
+                                                  TLI.getPointerTy()),
+                                      CfaArg);
+       setValue(&I, DAG.getNode(ISD::ADD,
+                                TLI.getPointerTy(),
+                                DAG.getNode(ISD::FRAMEADDR,
+                                            TLI.getPointerTy(),
+                                            DAG.getConstant(0,
+                                                            TLI.getPointerTy())),
+                                Offset));
+     } else {
+       setValue(&I, DAG.getConstant(0, TLI.getPointerTy()));
+     }
+
+     return 0;
+  }
+
+  case Intrinsic::sqrt:
     setValue(&I, DAG.getNode(ISD::FSQRT,
                              getValue(I.getOperand(1)).getValueType(),
                              getValue(I.getOperand(1))));
     return 0;
-  case Intrinsic::powi_f32:
-  case Intrinsic::powi_f64:
+  case Intrinsic::powi:
     setValue(&I, DAG.getNode(ISD::FPOWI,
                              getValue(I.getOperand(1)).getValueType(),
                              getValue(I.getOperand(1)),
                              getValue(I.getOperand(2))));
     return 0;
+  case Intrinsic::sin:
+    setValue(&I, DAG.getNode(ISD::FSIN,
+                             getValue(I.getOperand(1)).getValueType(),
+                             getValue(I.getOperand(1))));
+    return 0;
+  case Intrinsic::cos:
+    setValue(&I, DAG.getNode(ISD::FCOS,
+                             getValue(I.getOperand(1)).getValueType(),
+                             getValue(I.getOperand(1))));
+    return 0;
+  case Intrinsic::pow:
+    setValue(&I, DAG.getNode(ISD::FPOW,
+                             getValue(I.getOperand(1)).getValueType(),
+                             getValue(I.getOperand(1)),
+                             getValue(I.getOperand(2))));
+    return 0;
   case Intrinsic::pcmarker: {
     SDOperand Tmp = getValue(I.getOperand(1));
     DAG.setRoot(DAG.getNode(ISD::PCMARKER, MVT::Other, getRoot(), Tmp));
@@ -2769,10 +2849,6 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     SDOperand Arg = getValue(I.getOperand(1));
     MVT::ValueType Ty = Arg.getValueType();
     SDOperand result = DAG.getNode(ISD::CTTZ, Ty, Arg);
-    if (Ty < MVT::i32)
-      result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result);
-    else if (Ty > MVT::i32)
-      result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result);
     setValue(&I, result);
     return 0;
   }
@@ -2780,10 +2856,6 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     SDOperand Arg = getValue(I.getOperand(1));
     MVT::ValueType Ty = Arg.getValueType();
     SDOperand result = DAG.getNode(ISD::CTLZ, Ty, Arg);
-    if (Ty < MVT::i32)
-      result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result);
-    else if (Ty > MVT::i32)
-      result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result);
     setValue(&I, result);
     return 0;
   }
@@ -2791,10 +2863,6 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     SDOperand Arg = getValue(I.getOperand(1));
     MVT::ValueType Ty = Arg.getValueType();
     SDOperand result = DAG.getNode(ISD::CTPOP, Ty, Arg);
-    if (Ty < MVT::i32)
-      result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result);
-    else if (Ty > MVT::i32)
-      result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result);
     setValue(&I, result);
     return 0;
   }
@@ -2818,63 +2886,112 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
   case Intrinsic::var_annotation:
     // Discard annotate attributes
     return 0;
+
+  case Intrinsic::init_trampoline: {
+    const Function *F =
+      cast<Function>(IntrinsicInst::StripPointerCasts(I.getOperand(2)));
+
+    SDOperand Ops[6];
+    Ops[0] = getRoot();
+    Ops[1] = getValue(I.getOperand(1));
+    Ops[2] = getValue(I.getOperand(2));
+    Ops[3] = getValue(I.getOperand(3));
+    Ops[4] = DAG.getSrcValue(I.getOperand(1));
+    Ops[5] = DAG.getSrcValue(F);
+
+    SDOperand Tmp = DAG.getNode(ISD::TRAMPOLINE,
+                                DAG.getNodeValueTypes(TLI.getPointerTy(),
+                                                      MVT::Other), 2,
+                                Ops, 6);
+
+    setValue(&I, Tmp);
+    DAG.setRoot(Tmp.getValue(1));
+    return 0;
+  }
+
+  case Intrinsic::gcroot:
+    if (GCI) {
+      Value *Alloca = I.getOperand(1);
+      Constant *TypeMap = cast<Constant>(I.getOperand(2));
+      
+      FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).Val);
+      GCI->addStackRoot(FI->getIndex(), TypeMap);
+    }
+    return 0;
+
+  case Intrinsic::gcread:
+  case Intrinsic::gcwrite:
+    assert(0 && "Collector failed to lower gcread/gcwrite intrinsics!");
+    return 0;
+
+  case Intrinsic::flt_rounds: {
+    setValue(&I, DAG.getNode(ISD::FLT_ROUNDS, MVT::i32));
+    return 0;
+  }
+
+  case Intrinsic::trap: {
+    DAG.setRoot(DAG.getNode(ISD::TRAP, MVT::Other, getRoot()));
+    return 0;
+  }
   }
 }
 
 
-void SelectionDAGLowering::LowerCallTo(Instruction &I,
-                                       const Type *CalledValueTy,
-                                       unsigned CallingConv,
+void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee,
                                        bool IsTailCall,
-                                       SDOperand Callee, unsigned OpIdx,
                                        MachineBasicBlock *LandingPad) {
-  const PointerType *PT = cast<PointerType>(CalledValueTy);
+  const PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
   const FunctionType *FTy = cast<FunctionType>(PT->getElementType());
-  const ParamAttrsList *Attrs = FTy->getParamAttrs();
   MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
   unsigned BeginLabel = 0, EndLabel = 0;
-    
+
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
-  Args.reserve(I.getNumOperands());
-  for (unsigned i = OpIdx, e = I.getNumOperands(); i != e; ++i) {
-    Value *Arg = I.getOperand(i);
-    SDOperand ArgNode = getValue(Arg);
-    Entry.Node = ArgNode; Entry.Ty = Arg->getType();
-
-    unsigned attrInd = i - OpIdx + 1;
-    Entry.isSExt  = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::SExt);
-    Entry.isZExt  = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::ZExt);
-    Entry.isInReg = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::InReg);
-    Entry.isSRet  = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::StructRet);
+  Args.reserve(CS.arg_size());
+  for (CallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
+       i != e; ++i) {
+    SDOperand ArgNode = getValue(*i);
+    Entry.Node = ArgNode; Entry.Ty = (*i)->getType();
+
+    unsigned attrInd = i - CS.arg_begin() + 1;
+    Entry.isSExt  = CS.paramHasAttr(attrInd, ParamAttr::SExt);
+    Entry.isZExt  = CS.paramHasAttr(attrInd, ParamAttr::ZExt);
+    Entry.isInReg = CS.paramHasAttr(attrInd, ParamAttr::InReg);
+    Entry.isSRet  = CS.paramHasAttr(attrInd, ParamAttr::StructRet);
+    Entry.isNest  = CS.paramHasAttr(attrInd, ParamAttr::Nest);
+    Entry.isByVal = CS.paramHasAttr(attrInd, ParamAttr::ByVal);
     Args.push_back(Entry);
   }
 
-  if (ExceptionHandling && MMI) {
+  bool MarkTryRange = LandingPad ||
+    // C++ requires special handling of 'nounwind' calls.
+    (CS.doesNotThrow());
+
+  if (MarkTryRange && ExceptionHandling && MMI) {
     // Insert a label before the invoke call to mark the try range.  This can be
     // used to detect deletion of the invoke via the MachineModuleInfo.
     BeginLabel = MMI->NextLabelID();
     DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(),
                             DAG.getConstant(BeginLabel, MVT::i32)));
   }
-  
+
   std::pair<SDOperand,SDOperand> Result =
-    TLI.LowerCallTo(getRoot(), I.getType(), 
-                    Attrs && Attrs->paramHasAttr(0, ParamAttr::SExt),
-                    FTy->isVarArg(), CallingConv, IsTailCall, 
+    TLI.LowerCallTo(getRoot(), CS.getType(),
+                    CS.paramHasAttr(0, ParamAttr::SExt),
+                    FTy->isVarArg(), CS.getCallingConv(), IsTailCall,
                     Callee, Args, DAG);
-  if (I.getType() != Type::VoidTy)
-    setValue(&I, Result.first);
+  if (CS.getType() != Type::VoidTy)
+    setValue(CS.getInstruction(), Result.first);
   DAG.setRoot(Result.second);
 
-  if (ExceptionHandling && MMI) {
+  if (MarkTryRange && ExceptionHandling && MMI) {
     // Insert a label at the end of the invoke call to mark the try range.  This
     // can be used to detect deletion of the invoke via the MachineModuleInfo.
     EndLabel = MMI->NextLabelID();
     DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(),
                             DAG.getConstant(EndLabel, MVT::i32)));
 
-    // Inform MachineModuleInfo of range.    
+    // Inform MachineModuleInfo of range.
     MMI->addInvoke(LandingPad, BeginLabel, EndLabel);
   }
 }
@@ -2883,52 +3000,69 @@ void SelectionDAGLowering::LowerCallTo(Instruction &I,
 void SelectionDAGLowering::visitCall(CallInst &I) {
   const char *RenameFn = 0;
   if (Function *F = I.getCalledFunction()) {
-    if (F->isDeclaration())
+    if (F->isDeclaration()) {
       if (unsigned IID = F->getIntrinsicID()) {
         RenameFn = visitIntrinsicCall(I, IID);
         if (!RenameFn)
           return;
-      } else {    // Not an LLVM intrinsic.
-        const std::string &Name = F->getName();
-        if (Name[0] == 'c' && (Name == "copysign" || Name == "copysignf")) {
-          if (I.getNumOperands() == 3 &&   // Basic sanity checks.
-              I.getOperand(1)->getType()->isFloatingPoint() &&
-              I.getType() == I.getOperand(1)->getType() &&
-              I.getType() == I.getOperand(2)->getType()) {
-            SDOperand LHS = getValue(I.getOperand(1));
-            SDOperand RHS = getValue(I.getOperand(2));
-            setValue(&I, DAG.getNode(ISD::FCOPYSIGN, LHS.getValueType(),
-                                     LHS, RHS));
-            return;
-          }
-        } else if (Name[0] == 'f' && (Name == "fabs" || Name == "fabsf")) {
-          if (I.getNumOperands() == 2 &&   // Basic sanity checks.
-              I.getOperand(1)->getType()->isFloatingPoint() &&
-              I.getType() == I.getOperand(1)->getType()) {
-            SDOperand Tmp = getValue(I.getOperand(1));
-            setValue(&I, DAG.getNode(ISD::FABS, Tmp.getValueType(), Tmp));
-            return;
-          }
-        } else if (Name[0] == 's' && (Name == "sin" || Name == "sinf")) {
-          if (I.getNumOperands() == 2 &&   // Basic sanity checks.
-              I.getOperand(1)->getType()->isFloatingPoint() &&
-              I.getType() == I.getOperand(1)->getType()) {
-            SDOperand Tmp = getValue(I.getOperand(1));
-            setValue(&I, DAG.getNode(ISD::FSIN, Tmp.getValueType(), Tmp));
-            return;
-          }
-        } else if (Name[0] == 'c' && (Name == "cos" || Name == "cosf")) {
-          if (I.getNumOperands() == 2 &&   // Basic sanity checks.
-              I.getOperand(1)->getType()->isFloatingPoint() &&
-              I.getType() == I.getOperand(1)->getType()) {
-            SDOperand Tmp = getValue(I.getOperand(1));
-            setValue(&I, DAG.getNode(ISD::FCOS, Tmp.getValueType(), Tmp));
-            return;
-          }
+      }
+    }
+
+    // Check for well-known libc/libm calls.  If the function is internal, it
+    // can't be a library call.
+    unsigned NameLen = F->getNameLen();
+    if (!F->hasInternalLinkage() && NameLen) {
+      const char *NameStr = F->getNameStart();
+      if (NameStr[0] == 'c' &&
+          ((NameLen == 8 && !strcmp(NameStr, "copysign")) ||
+           (NameLen == 9 && !strcmp(NameStr, "copysignf")))) {
+        if (I.getNumOperands() == 3 &&   // Basic sanity checks.
+            I.getOperand(1)->getType()->isFloatingPoint() &&
+            I.getType() == I.getOperand(1)->getType() &&
+            I.getType() == I.getOperand(2)->getType()) {
+          SDOperand LHS = getValue(I.getOperand(1));
+          SDOperand RHS = getValue(I.getOperand(2));
+          setValue(&I, DAG.getNode(ISD::FCOPYSIGN, LHS.getValueType(),
+                                   LHS, RHS));
+          return;
+        }
+      } else if (NameStr[0] == 'f' &&
+                 ((NameLen == 4 && !strcmp(NameStr, "fabs")) ||
+                  (NameLen == 5 && !strcmp(NameStr, "fabsf")) ||
+                  (NameLen == 5 && !strcmp(NameStr, "fabsl")))) {
+        if (I.getNumOperands() == 2 &&   // Basic sanity checks.
+            I.getOperand(1)->getType()->isFloatingPoint() &&
+            I.getType() == I.getOperand(1)->getType()) {
+          SDOperand Tmp = getValue(I.getOperand(1));
+          setValue(&I, DAG.getNode(ISD::FABS, Tmp.getValueType(), Tmp));
+          return;
+        }
+      } else if (NameStr[0] == 's' && 
+                 ((NameLen == 3 && !strcmp(NameStr, "sin")) ||
+                  (NameLen == 4 && !strcmp(NameStr, "sinf")) ||
+                  (NameLen == 4 && !strcmp(NameStr, "sinl")))) {
+        if (I.getNumOperands() == 2 &&   // Basic sanity checks.
+            I.getOperand(1)->getType()->isFloatingPoint() &&
+            I.getType() == I.getOperand(1)->getType()) {
+          SDOperand Tmp = getValue(I.getOperand(1));
+          setValue(&I, DAG.getNode(ISD::FSIN, Tmp.getValueType(), Tmp));
+          return;
+        }
+      } else if (NameStr[0] == 'c' &&
+                 ((NameLen == 3 && !strcmp(NameStr, "cos")) ||
+                  (NameLen == 4 && !strcmp(NameStr, "cosf")) ||
+                  (NameLen == 4 && !strcmp(NameStr, "cosl")))) {
+        if (I.getNumOperands() == 2 &&   // Basic sanity checks.
+            I.getOperand(1)->getType()->isFloatingPoint() &&
+            I.getType() == I.getOperand(1)->getType()) {
+          SDOperand Tmp = getValue(I.getOperand(1));
+          setValue(&I, DAG.getNode(ISD::FCOS, Tmp.getValueType(), Tmp));
+          return;
         }
       }
+    }
   } else if (isa<InlineAsm>(I.getOperand(0))) {
-    visitInlineAsm(I);
+    visitInlineAsm(&I);
     return;
   }
 
@@ -2938,11 +3072,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) {
   else
     Callee = DAG.getExternalSymbol(RenameFn, TLI.getPointerTy());
 
-  LowerCallTo(I, I.getCalledValue()->getType(),
-              I.getCallingConv(),
-              I.isTailCall(),
-              Callee,
-              1);
+  LowerCallTo(&I, Callee, I.isTailCall());
 }
 
 
@@ -2952,11 +3082,6 @@ void SelectionDAGLowering::visitCall(CallInst &I) {
 /// If the Flag pointer is NULL, no flag is used.
 SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
                                         SDOperand &Chain, SDOperand *Flag)const{
-  // Get the list of registers, in the appropriate order.
-  std::vector<unsigned> R(Regs);
-  if (!DAG.getTargetLoweringInfo().isLittleEndian())
-    std::reverse(R.begin(), R.end());
-
   // Copy the legal parts from the registers.
   unsigned NumParts = Regs.size();
   SmallVector<SDOperand, 8> Parts(NumParts);
@@ -2971,7 +3096,7 @@ SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
   }
   
   // Assemble the legal parts into the final value.
-  return getCopyFromParts(DAG, &Parts[0], NumParts, RegVT, ValueVT, false);
+  return getCopyFromParts(DAG, &Parts[0], NumParts, RegVT, ValueVT);
 }
 
 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
@@ -2980,21 +3105,16 @@ SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
 /// If the Flag pointer is NULL, no flag is used.
 void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG,
                                  SDOperand &Chain, SDOperand *Flag) const {
-  // Get the list of registers, in the appropriate order.
-  std::vector<unsigned> R(Regs);
-  if (!DAG.getTargetLoweringInfo().isLittleEndian())
-    std::reverse(R.begin(), R.end());
-
   // Get the list of the values's legal parts.
   unsigned NumParts = Regs.size();
   SmallVector<SDOperand, 8> Parts(NumParts);
-  getCopyToParts(DAG, Val, &Parts[0], NumParts, RegVT, false);
+  getCopyToParts(DAG, Val, &Parts[0], NumParts, RegVT);
 
   // Copy the parts into the registers.
   for (unsigned i = 0; i != NumParts; ++i) {
     SDOperand Part = Flag ?
-                     DAG.getCopyToReg(Chain, R[i], Parts[i], *Flag) :
-                     DAG.getCopyToReg(Chain, R[i], Parts[i]);
+                     DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag) :
+                     DAG.getCopyToReg(Chain, Regs[i], Parts[i]);
     Chain = Part.getValue(0);
     if (Flag)
       *Flag = Part.getValue(1);
@@ -3250,9 +3370,9 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber,
         ValueVT = RegVT;
 
       // Create the appropriate number of virtual registers.
-      SSARegMap *RegMap = MF.getSSARegMap();
+      MachineRegisterInfo &RegInfo = MF.getRegInfo();
       for (; NumRegs; --NumRegs)
-        Regs.push_back(RegMap->createVirtualRegister(PhysReg.second));
+        Regs.push_back(RegInfo.createVirtualRegister(PhysReg.second));
       
       OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
       OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs);
@@ -3319,8 +3439,8 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber,
 
 /// visitInlineAsm - Handle a call to an InlineAsm object.
 ///
-void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
-  InlineAsm *IA = cast<InlineAsm>(I.getOperand(0));
+void SelectionDAGLowering::visitInlineAsm(CallSite CS) {
+  InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
 
   /// ConstraintOperands - Information about all of the constraints.
   std::vector<AsmOperandInfo> ConstraintOperands;
@@ -3340,7 +3460,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
   // registers, because it will not know to avoid the earlyclobbered output reg.
   bool SawEarlyClobber = false;
   
-  unsigned OpNo = 1;   // OpNo - The operand of the CallInst.
+  unsigned ArgNo = 0;   // ArgNo - The argument of the CallInst.
   for (unsigned i = 0, e = ConstraintInfos.size(); i != e; ++i) {
     ConstraintOperands.push_back(AsmOperandInfo(ConstraintInfos[i]));
     AsmOperandInfo &OpInfo = ConstraintOperands.back();
@@ -3353,14 +3473,14 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
       if (!OpInfo.isIndirect) {
         // The return value of the call is this value.  As such, there is no
         // corresponding argument.
-        assert(I.getType() != Type::VoidTy && "Bad inline asm!");
-        OpVT = TLI.getValueType(I.getType());
+        assert(CS.getType() != Type::VoidTy && "Bad inline asm!");
+        OpVT = TLI.getValueType(CS.getType());
       } else {
-        OpInfo.CallOperandVal = I.getOperand(OpNo++);
+        OpInfo.CallOperandVal = CS.getArgument(ArgNo++);
       }
       break;
     case InlineAsm::isInput:
-      OpInfo.CallOperandVal = I.getOperand(OpNo++);
+      OpInfo.CallOperandVal = CS.getArgument(ArgNo++);
       break;
     case InlineAsm::isClobber:
       // Nothing to do.
@@ -3368,31 +3488,37 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
     }
 
     // If this is an input or an indirect output, process the call argument.
+    // BasicBlocks are labels, currently appearing only in asm's.
     if (OpInfo.CallOperandVal) {
-      OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
-      const Type *OpTy = OpInfo.CallOperandVal->getType();
-      // If this is an indirect operand, the operand is a pointer to the
-      // accessed type.
-      if (OpInfo.isIndirect)
-        OpTy = cast<PointerType>(OpTy)->getElementType();
-      
-      // If OpTy is not a first-class value, it may be a struct/union that we
-      // can tile with integers.
-      if (!OpTy->isFirstClassType() && OpTy->isSized()) {
-        unsigned BitSize = TD->getTypeSizeInBits(OpTy);
-        switch (BitSize) {
-        default: break;
-        case 1:
-        case 8:
-        case 16:
-        case 32:
-        case 64:
-          OpTy = IntegerType::get(BitSize);
-          break;
+      if (isa<BasicBlock>(OpInfo.CallOperandVal))
+        OpInfo.CallOperand = 
+          DAG.getBasicBlock(FuncInfo.MBBMap[cast<BasicBlock>(OpInfo.CallOperandVal)]);
+      else {
+        OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
+        const Type *OpTy = OpInfo.CallOperandVal->getType();
+        // If this is an indirect operand, the operand is a pointer to the
+        // accessed type.
+        if (OpInfo.isIndirect)
+          OpTy = cast<PointerType>(OpTy)->getElementType();
+
+        // If OpTy is not a first-class value, it may be a struct/union that we
+        // can tile with integers.
+        if (!OpTy->isFirstClassType() && OpTy->isSized()) {
+          unsigned BitSize = TD->getTypeSizeInBits(OpTy);
+          switch (BitSize) {
+          default: break;
+          case 1:
+          case 8:
+          case 16:
+          case 32:
+          case 64:
+            OpTy = IntegerType::get(BitSize);
+            break;
+          }
         }
+
+        OpVT = TLI.getValueType(OpTy, true);
       }
-      
-      OpVT = TLI.getValueType(OpTy, true);
     }
     
     OpInfo.ConstraintVT = OpVT;
@@ -3425,7 +3551,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
         // Otherwise, create a stack slot and emit a store to it before the
         // asm.
         const Type *Ty = OpVal->getType();
-        uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+        uint64_t TySize = TLI.getTargetData()->getABITypeSize(Ty);
         unsigned Align  = TLI.getTargetData()->getPrefTypeAlignment(Ty);
         MachineFunction &MF = DAG.getMachineFunction();
         int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align);
@@ -3505,7 +3631,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
         // This is the result value of the call.
         assert(RetValRegs.Regs.empty() &&
                "Cannot have multiple output constraints yet!");
-        assert(I.getType() != Type::VoidTy && "Bad inline asm!");
+        assert(CS.getType() != Type::VoidTy && "Bad inline asm!");
         RetValRegs = OpInfo.AssignedRegs;
       } else {
         IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
@@ -3566,20 +3692,20 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
         assert(!OpInfo.isIndirect && 
                "Don't know how to handle indirect other inputs yet!");
         
-        InOperandVal = TLI.isOperandValidForConstraint(InOperandVal,
-                                                       OpInfo.ConstraintCode[0],
-                                                       DAG);
-        if (!InOperandVal.Val) {
+        std::vector<SDOperand> Ops;
+        TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode[0],
+                                         Ops, DAG);
+        if (Ops.empty()) {
           cerr << "Invalid operand for inline asm constraint '"
                << OpInfo.ConstraintCode << "'!\n";
           exit(1);
         }
         
         // Add information to the INLINEASM node to know about this input.
-        unsigned ResOpType = 3 /*IMM*/ | (1 << 3);
+        unsigned ResOpType = 3 /*IMM*/ | (Ops.size() << 3);
         AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, 
                                                         TLI.getPointerTy()));
-        AsmNodeOperands.push_back(InOperandVal);
+        AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
         break;
       } else if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
         assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
@@ -3639,13 +3765,13 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
     // width/num elts.  Make sure to convert it to the right type with
     // bit_convert.
     if (MVT::isVector(Val.getValueType())) {
-      const VectorType *VTy = cast<VectorType>(I.getType());
+      const VectorType *VTy = cast<VectorType>(CS.getType());
       MVT::ValueType DesiredVT = TLI.getValueType(VTy);
       
       Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val);
     }
     
-    setValue(&I, Val);
+    setValue(CS.getInstruction(), Val);
   }
   
   std::vector<std::pair<SDOperand, Value*> > StoresToEmit;
@@ -3683,9 +3809,9 @@ void SelectionDAGLowering::visitMalloc(MallocInst &I) {
     Src = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, Src);
 
   // Scale the source by the type size.
-  uint64_t ElementSize = TD->getTypeSize(I.getType()->getElementType());
+  uint64_t ElementSize = TD->getABITypeSize(I.getType()->getElementType());
   Src = DAG.getNode(ISD::MUL, Src.getValueType(),
-                    Src, getIntPtrConstant(ElementSize));
+                    Src, DAG.getIntPtrConstant(ElementSize));
 
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
@@ -3762,8 +3888,6 @@ void SelectionDAGLowering::visitVACopy(CallInst &I) {
 /// integrated into SDISel.
 std::vector<SDOperand> 
 TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
-  const FunctionType *FTy = F.getFunctionType();
-  const ParamAttrsList *Attrs = FTy->getParamAttrs();
   // Add CC# and isVararg as operands to the FORMAL_ARGUMENTS node.
   std::vector<SDOperand> Ops;
   Ops.push_back(DAG.getRoot());
@@ -3782,16 +3906,25 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
 
     // FIXME: Distinguish between a formal with no [sz]ext attribute from one
     // that is zero extended!
-    if (Attrs && Attrs->paramHasAttr(j, ParamAttr::ZExt))
+    if (F.paramHasAttr(j, ParamAttr::ZExt))
       Flags &= ~(ISD::ParamFlags::SExt);
-    if (Attrs && Attrs->paramHasAttr(j, ParamAttr::SExt))
+    if (F.paramHasAttr(j, ParamAttr::SExt))
       Flags |= ISD::ParamFlags::SExt;
-    if (Attrs && Attrs->paramHasAttr(j, ParamAttr::InReg))
+    if (F.paramHasAttr(j, ParamAttr::InReg))
       Flags |= ISD::ParamFlags::InReg;
-    if (Attrs && Attrs->paramHasAttr(j, ParamAttr::StructRet))
+    if (F.paramHasAttr(j, ParamAttr::StructRet))
       Flags |= ISD::ParamFlags::StructReturn;
-    if (Attrs && Attrs->paramHasAttr(j, ParamAttr::ByVal))
+    if (F.paramHasAttr(j, ParamAttr::ByVal)) {
       Flags |= ISD::ParamFlags::ByVal;
+      const PointerType *Ty = cast<PointerType>(I->getType());
+      const Type *ElementTy = Ty->getElementType();
+      unsigned FrameAlign = Log2_32(getByValTypeAlignment(ElementTy));
+      unsigned FrameSize  = getTargetData()->getABITypeSize(ElementTy);
+      Flags |= (FrameAlign << ISD::ParamFlags::ByValAlignOffs);
+      Flags |= (FrameSize  << ISD::ParamFlags::ByValSizeOffs);
+    }
+    if (F.paramHasAttr(j, ParamAttr::Nest))
+      Flags |= ISD::ParamFlags::Nest;
     Flags |= (OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs);
     
     switch (getTypeAction(VT)) {
@@ -3847,16 +3980,16 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
     case Promote: {
       SDOperand Op(Result, i++);
       if (MVT::isInteger(VT)) {
-        if (Attrs && Attrs->paramHasAttr(Idx, ParamAttr::SExt))
+        if (F.paramHasAttr(Idx, ParamAttr::SExt))
           Op = DAG.getNode(ISD::AssertSext, Op.getValueType(), Op,
                            DAG.getValueType(VT));
-        else if (Attrs && Attrs->paramHasAttr(Idx, ParamAttr::ZExt))
+        else if (F.paramHasAttr(Idx, ParamAttr::ZExt))
           Op = DAG.getNode(ISD::AssertZext, Op.getValueType(), Op,
                            DAG.getValueType(VT));
         Op = DAG.getNode(ISD::TRUNCATE, VT, Op);
       } else {
         assert(MVT::isFloatingPoint(VT) && "Not int or FP?");
-        Op = DAG.getNode(ISD::FP_ROUND, VT, Op);
+        Op = DAG.getNode(ISD::FP_ROUND, VT, Op, DAG.getIntPtrConstant(1));
       }
       Ops.push_back(Op);
       break;
@@ -3867,7 +4000,7 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
       SmallVector<SDOperand, 4> Parts(NumParts);
       for (unsigned j = 0; j != NumParts; ++j)
         Parts[j] = SDOperand(Result, i++);
-      Ops.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT, true));
+      Ops.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT));
       break;
     }
     }
@@ -3910,6 +4043,17 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
       Flags |= ISD::ParamFlags::InReg;
     if (Args[i].isSRet)
       Flags |= ISD::ParamFlags::StructReturn;
+    if (Args[i].isByVal) {
+      Flags |= ISD::ParamFlags::ByVal;
+      const PointerType *Ty = cast<PointerType>(Args[i].Ty);
+      const Type *ElementTy = Ty->getElementType();
+      unsigned FrameAlign = Log2_32(getByValTypeAlignment(ElementTy));
+      unsigned FrameSize  = getTargetData()->getABITypeSize(ElementTy);
+      Flags |= (FrameAlign << ISD::ParamFlags::ByValAlignOffs);
+      Flags |= (FrameSize  << ISD::ParamFlags::ByValSizeOffs);
+    }
+    if (Args[i].isNest)
+      Flags |= ISD::ParamFlags::Nest;
     Flags |= OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs;
     
     switch (getTypeAction(VT)) {
@@ -3939,7 +4083,7 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
       MVT::ValueType PartVT = getRegisterType(VT);
       unsigned NumParts = getNumRegisters(VT);
       SmallVector<SDOperand, 4> Parts(NumParts);
-      getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT, true);
+      getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT);
       for (unsigned i = 0; i != NumParts; ++i) {
         // if it isn't first piece, alignment must be 1
         unsigned MyFlags = Flags;
@@ -3969,7 +4113,7 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
   SDOperand Res = DAG.getNode(ISD::CALL,
                               DAG.getVTList(&RetTys[0], NumRegs + 1),
                               &Ops[0], Ops.size());
-  SDOperand Chain = Res.getValue(NumRegs);
+  Chain = Res.getValue(NumRegs);
 
   // Gather up the call result into a single value.
   if (RetTy != Type::VoidTy) {
@@ -3979,7 +4123,7 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
     SmallVector<SDOperand, 4> Results(NumRegs);
     for (unsigned i = 0; i != NumRegs; ++i)
       Results[i] = Res.getValue(i);
-    Res = getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, false, AssertOp);
+    Res = getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, AssertOp);
   }
 
   return std::make_pair(Res, Chain);
@@ -4113,6 +4257,17 @@ void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) {
   unsigned Align = (unsigned)cast<ConstantSDNode>(Op4)->getValue();
   if (Align == 0) Align = 1;
 
+  // If the source and destination are known to not be aliases, we can
+  // lower memmove as memcpy.
+  if (Op == ISD::MEMMOVE) {
+    uint64_t Size = -1ULL;
+    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op3))
+      Size = C->getValue();
+    if (AA.alias(I.getOperand(1), Size, I.getOperand(2), Size) ==
+        AliasAnalysis::NoAlias)
+      Op = ISD::MEMCPY;
+  }
+
   if (ConstantSDNode *Size = dyn_cast<ConstantSDNode>(Op3)) {
     std::vector<MVT::ValueType> MemOps;
 
@@ -4181,13 +4336,13 @@ void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) {
                            I.getOperand(1), DstOff);
           } else {
             Value = DAG.getLoad(VT, getRoot(),
-                        getMemBasePlusOffset(Op2, SrcOff, DAG, TLI),
-                        I.getOperand(2), SrcOff);
+                                getMemBasePlusOffset(Op2, SrcOff, DAG, TLI),
+                                I.getOperand(2), SrcOff, false, Align);
             Chain = Value.getValue(1);
             Store =
               DAG.getStore(Chain, Value,
                            getMemBasePlusOffset(Op1, DstOff, DAG, TLI),
-                           I.getOperand(1), DstOff);
+                           I.getOperand(1), DstOff, false, Align);
           }
           OutChains.push_back(Store);
           SrcOff += VTSize;
@@ -4205,7 +4360,22 @@ void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) {
     }
   }
 
-  DAG.setRoot(DAG.getNode(Op, MVT::Other, getRoot(), Op1, Op2, Op3, Op4));
+  SDOperand AlwaysInline = DAG.getConstant(0, MVT::i1);
+  SDOperand Node;
+  switch(Op) {
+    default:
+      assert(0 && "Unknown Op");
+    case ISD::MEMCPY:
+      Node = DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline);
+      break;
+    case ISD::MEMMOVE:
+      Node = DAG.getMemmove(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline);
+      break;
+    case ISD::MEMSET:
+      Node = DAG.getMemset(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline);
+      break;
+  }
+  DAG.setRoot(Node);
 }
 
 //===----------------------------------------------------------------------===//
@@ -4213,19 +4383,27 @@ void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) {
 //===----------------------------------------------------------------------===//
 
 unsigned SelectionDAGISel::MakeReg(MVT::ValueType VT) {
-  return RegMap->createVirtualRegister(TLI.getRegClassFor(VT));
+  return RegInfo->createVirtualRegister(TLI.getRegClassFor(VT));
 }
 
 void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<AliasAnalysis>();
+  AU.addRequired<CollectorModuleMetadata>();
   AU.setPreservesAll();
 }
 
 
 
 bool SelectionDAGISel::runOnFunction(Function &Fn) {
+  // Get alias analysis for load/store combining.
+  AA = &getAnalysis<AliasAnalysis>();
+
   MachineFunction &MF = MachineFunction::construct(&Fn, TLI.getTargetMachine());
-  RegMap = MF.getSSARegMap();
+  if (MF.getFunction()->hasCollector())
+    GCI = &getAnalysis<CollectorModuleMetadata>().get(*MF.getFunction());
+  else
+    GCI = 0;
+  RegInfo = &MF.getRegInfo();
   DOUT << "\n\n\n=== " << Fn.getName() << "\n";
 
   FunctionLoweringInfo FuncInfo(TLI, Fn, MF);
@@ -4242,9 +4420,9 @@ bool SelectionDAGISel::runOnFunction(Function &Fn) {
   // Add function live-ins to entry block live-in set.
   BasicBlock *EntryBB = &Fn.getEntryBlock();
   BB = FuncInfo.MBBMap[EntryBB];
-  if (!MF.livein_empty())
-    for (MachineFunction::livein_iterator I = MF.livein_begin(),
-           E = MF.livein_end(); I != E; ++I)
+  if (!RegInfo->livein_empty())
+    for (MachineRegisterInfo::livein_iterator I = RegInfo->livein_begin(),
+           E = RegInfo->livein_end(); I != E; ++I)
       BB->addLiveIn(I->first);
 
 #ifndef NDEBUG
@@ -4269,7 +4447,7 @@ SDOperand SelectionDAGLowering::CopyValueToVirtualRegister(Value *V,
   SmallVector<SDOperand, 8> Chains(NumRegs);
 
   // Copy the value by legal parts into sequential virtual registers.
-  getCopyToParts(DAG, Op, &Regs[0], NumRegs, RegisterVT, false);
+  getCopyToParts(DAG, Op, &Regs[0], NumRegs, RegisterVT);
   for (unsigned i = 0; i != NumRegs; ++i)
     Chains[i] = DAG.getCopyToReg(getRoot(), Reg + i, Regs[i]);
   return DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs);
@@ -4306,22 +4484,63 @@ LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL,
 
 static void copyCatchInfo(BasicBlock *SrcBB, BasicBlock *DestBB,
                           MachineModuleInfo *MMI, FunctionLoweringInfo &FLI) {
-  assert(!FLI.MBBMap[SrcBB]->isLandingPad() &&
-         "Copying catch info out of a landing pad!");
   for (BasicBlock::iterator I = SrcBB->begin(), E = --SrcBB->end(); I != E; ++I)
     if (isSelector(I)) {
       // Apply the catch info to DestBB.
       addCatchInfo(cast<CallInst>(*I), MMI, FLI.MBBMap[DestBB]);
 #ifndef NDEBUG
-      FLI.CatchInfoFound.insert(I);
+      if (!FLI.MBBMap[SrcBB]->isLandingPad())
+        FLI.CatchInfoFound.insert(I);
 #endif
     }
 }
 
+/// CheckDAGForTailCallsAndFixThem - This Function looks for CALL nodes in the
+/// DAG and fixes their tailcall attribute operand.
+static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG, 
+                                           TargetLowering& TLI) {
+  SDNode * Ret = NULL;
+  SDOperand Terminator = DAG.getRoot();
+
+  // Find RET node.
+  if (Terminator.getOpcode() == ISD::RET) {
+    Ret = Terminator.Val;
+  }
+ 
+  // Fix tail call attribute of CALL nodes.
+  for (SelectionDAG::allnodes_iterator BE = DAG.allnodes_begin(),
+         BI = prior(DAG.allnodes_end()); BI != BE; --BI) {
+    if (BI->getOpcode() == ISD::CALL) {
+      SDOperand OpRet(Ret, 0);
+      SDOperand OpCall(static_cast<SDNode*>(BI), 0);
+      bool isMarkedTailCall = 
+        cast<ConstantSDNode>(OpCall.getOperand(3))->getValue() != 0;
+      // If CALL node has tail call attribute set to true and the call is not
+      // eligible (no RET or the target rejects) the attribute is fixed to
+      // false. The TargetLowering::IsEligibleForTailCallOptimization function
+      // must correctly identify tail call optimizable calls.
+      if (isMarkedTailCall && 
+          (Ret==NULL || 
+           !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG))) {
+        SmallVector<SDOperand, 32> Ops;
+        unsigned idx=0;
+        for(SDNode::op_iterator I =OpCall.Val->op_begin(), 
+              E=OpCall.Val->op_end(); I!=E; I++, idx++) {
+          if (idx!=3)
+            Ops.push_back(*I);
+          else 
+            Ops.push_back(DAG.getConstant(false, TLI.getPointerTy()));
+        }
+        DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size());
+      }
+    }
+  }
+}
+
 void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
        std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
                                          FunctionLoweringInfo &FuncInfo) {
-  SelectionDAGLowering SDL(DAG, TLI, FuncInfo);
+  SelectionDAGLowering SDL(DAG, TLI, *AA, FuncInfo, GCI);
 
   std::vector<SDOperand> UnorderedChains;
 
@@ -4406,8 +4625,8 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
   if (TI->getNumSuccessors())
     SuccsHandled.resize(BB->getParent()->getNumBlockIDs());
     
-  // Check successor nodes PHI nodes that expect a constant to be available from
-  // this block.
+  // Check successor nodes' PHI nodes that expect a constant to be available
+  // from this block.
   for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
     BasicBlock *SuccBB = TI->getSuccessor(succ);
     if (!isa<PHINode>(SuccBB->begin())) continue;
@@ -4495,28 +4714,41 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
     
   // Make sure the root of the DAG is up-to-date.
   DAG.setRoot(SDL.getRoot());
+
+  // Check whether calls in this block are real tail calls. Fix up CALL nodes
+  // with correct tailcall attribute so that the target can rely on the tailcall
+  // attribute indicating whether the call is really eligible for tail call
+  // optimization.
+  CheckDAGForTailCallsAndFixThem(DAG, TLI);
 }
 
 void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) {
-  // Get alias analysis for load/store combining.
-  AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
+  DOUT << "Lowered selection DAG:\n";
+  DEBUG(DAG.dump());
 
   // Run the DAG combiner in pre-legalize mode.
-  DAG.Combine(false, AA);
+  DAG.Combine(false, *AA);
   
-  DOUT << "Lowered selection DAG:\n";
+  DOUT << "Optimized lowered selection DAG:\n";
   DEBUG(DAG.dump());
   
   // Second step, hack on the DAG until it only uses operations and types that
   // the target supports.
+#if 0  // Enable this some day.
+  DAG.LegalizeTypes();
+  // Someday even later, enable a dag combine pass here.
+#endif
   DAG.Legalize();
   
   DOUT << "Legalized selection DAG:\n";
   DEBUG(DAG.dump());
   
   // Run the DAG combiner in post-legalize mode.
-  DAG.Combine(true, AA);
+  DAG.Combine(true, *AA);
   
+  DOUT << "Optimized legalized selection DAG:\n";
+  DEBUG(DAG.dump());
+
   if (ViewISelDAGs) DAG.viewGraph();
 
   // Third, instruction select all of the operations to machine code, adding the
@@ -4555,8 +4787,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
       MachineInstr *PHI = PHINodesToUpdate[i].first;
       assert(PHI->getOpcode() == TargetInstrInfo::PHI &&
              "This is not a machine PHI node that we are updating!");
-      PHI->addRegOperand(PHINodesToUpdate[i].second, false);
-      PHI->addMachineBasicBlockOperand(BB);
+      PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[i].second,
+                                                false));
+      PHI->addOperand(MachineOperand::CreateMBB(BB));
     }
     return;
   }
@@ -4566,7 +4799,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
     if (!BitTestCases[i].Emitted) {
       SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
       CurDAG = &HSDAG;
-      SelectionDAGLowering HSDL(HSDAG, TLI, FuncInfo);    
+      SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI);
       // Set the current basic block to the mbb we wish to insert the code into
       BB = BitTestCases[i].Parent;
       HSDL.setCurrentBasicBlock(BB);
@@ -4579,7 +4812,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
     for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) {
       SelectionDAG BSDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
       CurDAG = &BSDAG;
-      SelectionDAGLowering BSDL(BSDAG, TLI, FuncInfo);
+      SelectionDAGLowering BSDL(BSDAG, TLI, *AA, FuncInfo, GCI);
       // Set the current basic block to the mbb we wish to insert the code into
       BB = BitTestCases[i].Cases[j].ThisBB;
       BSDL.setCurrentBasicBlock(BB);
@@ -4607,18 +4840,22 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
       // This is "default" BB. We have two jumps to it. From "header" BB and
       // from last "case" BB.
       if (PHIBB == BitTestCases[i].Default) {
-        PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
-        PHI->addMachineBasicBlockOperand(BitTestCases[i].Parent);
-        PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
-        PHI->addMachineBasicBlockOperand(BitTestCases[i].Cases.back().ThisBB);
+        PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second,
+                                                  false));
+        PHI->addOperand(MachineOperand::CreateMBB(BitTestCases[i].Parent));
+        PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second,
+                                                  false));
+        PHI->addOperand(MachineOperand::CreateMBB(BitTestCases[i].Cases.
+                                                  back().ThisBB));
       }
       // One of "cases" BB.
       for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) {
         MachineBasicBlock* cBB = BitTestCases[i].Cases[j].ThisBB;
         if (cBB->succ_end() !=
             std::find(cBB->succ_begin(),cBB->succ_end(), PHIBB)) {
-          PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
-          PHI->addMachineBasicBlockOperand(cBB);
+          PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second,
+                                                    false));
+          PHI->addOperand(MachineOperand::CreateMBB(cBB));
         }
       }
     }
@@ -4632,7 +4869,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
     if (!JTCases[i].first.Emitted) {
       SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
       CurDAG = &HSDAG;
-      SelectionDAGLowering HSDL(HSDAG, TLI, FuncInfo);    
+      SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI);
       // Set the current basic block to the mbb we wish to insert the code into
       BB = JTCases[i].first.HeaderBB;
       HSDL.setCurrentBasicBlock(BB);
@@ -4644,7 +4881,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
     
     SelectionDAG JSDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
     CurDAG = &JSDAG;
-    SelectionDAGLowering JSDL(JSDAG, TLI, FuncInfo);
+    SelectionDAGLowering JSDL(JSDAG, TLI, *AA, FuncInfo, GCI);
     // Set the current basic block to the mbb we wish to insert the code into
     BB = JTCases[i].second.MBB;
     JSDL.setCurrentBasicBlock(BB);
@@ -4661,13 +4898,15 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
              "This is not a machine PHI node that we are updating!");
       // "default" BB. We can go there only from header BB.
       if (PHIBB == JTCases[i].second.Default) {
-        PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
-        PHI->addMachineBasicBlockOperand(JTCases[i].first.HeaderBB);
+        PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second,
+                                                  false));
+        PHI->addOperand(MachineOperand::CreateMBB(JTCases[i].first.HeaderBB));
       }
       // JT BB. Just iterate over successors here
       if (BB->succ_end() != std::find(BB->succ_begin(),BB->succ_end(), PHIBB)) {
-        PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
-        PHI->addMachineBasicBlockOperand(BB);
+        PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second,
+                                                  false));
+        PHI->addOperand(MachineOperand::CreateMBB(BB));
       }
     }
   }
@@ -4679,8 +4918,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
     assert(PHI->getOpcode() == TargetInstrInfo::PHI &&
            "This is not a machine PHI node that we are updating!");
     if (BB->isSuccessor(PHI->getParent())) {
-      PHI->addRegOperand(PHINodesToUpdate[i].second, false);
-      PHI->addMachineBasicBlockOperand(BB);
+      PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[i].second,
+                                                false));
+      PHI->addOperand(MachineOperand::CreateMBB(BB));
     }
   }
   
@@ -4689,7 +4929,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
   for (unsigned i = 0, e = SwitchCases.size(); i != e; ++i) {
     SelectionDAG SDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
     CurDAG = &SDAG;
-    SelectionDAGLowering SDL(SDAG, TLI, FuncInfo);
+    SelectionDAGLowering SDL(SDAG, TLI, *AA, FuncInfo, GCI);
     
     // Set the current basic block to the mbb we wish to insert the code into
     BB = SwitchCases[i].ThisBB;
@@ -4711,8 +4951,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
         for (unsigned pn = 0; ; ++pn) {
           assert(pn != PHINodesToUpdate.size() && "Didn't find PHI entry!");
           if (PHINodesToUpdate[pn].first == Phi) {
-            Phi->addRegOperand(PHINodesToUpdate[pn].second, false);
-            Phi->addMachineBasicBlockOperand(SwitchCases[i].ThisBB);
+            Phi->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pn].
+                                                      second, false));
+            Phi->addOperand(MachineOperand::CreateMBB(SwitchCases[i].ThisBB));
             break;
           }
         }
@@ -4746,6 +4987,9 @@ void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) {
   
   ScheduleDAG *SL = Ctor(this, &DAG, BB);
   BB = SL->Run();
+
+  if (ViewSUnitDAGs) SL->viewGraph();
+
   delete SL;
 }
 
@@ -4764,7 +5008,7 @@ HazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() {
 /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
 /// specified in the .td file (e.g. 255).
 bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, 
-                                    int64_t DesiredMaskS) {
+                                    int64_t DesiredMaskS) const {
   uint64_t ActualMask = RHS->getValue();
   uint64_t DesiredMask =DesiredMaskS & MVT::getIntVTBitMask(LHS.getValueType());
   
@@ -4793,7 +5037,7 @@ bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS,
 /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
 /// specified in the .td file (e.g. 255).
 bool SelectionDAGISel::CheckOrMask(SDOperand LHS, ConstantSDNode *RHS, 
-                                    int64_t DesiredMaskS) {
+                                    int64_t DesiredMaskS) const {
   uint64_t ActualMask = RHS->getValue();
   uint64_t DesiredMask =DesiredMaskS & MVT::getIntVTBitMask(LHS.getValueType());