X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAG.cpp;h=979dea0d7f0dbeaed408ea79a2126be25ab6d63b;hb=47d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7;hp=f8c11456d971a55699ce020d92dcd2a4b9d44d20;hpb=fed90b6d097d50881afb45e4d79f430db66dd741;p=oota-llvm.git

diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f8c11456d97..979dea0d7f0 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -24,12 +24,14 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
-#include "llvm/Support/MathExtras.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
@@ -68,7 +70,7 @@ SelectionDAG::DAGUpdateListener::~DAGUpdateListener() {}
 /// As such, this method can be used to do an exact bit-for-bit comparison of
 /// two floating point values.
 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
-  return Value.bitwiseIsEqual(V);
+  return getValueAPF().bitwiseIsEqual(V);
 }
 
 bool ConstantFPSDNode::isValueValidForType(MVT VT,
@@ -82,8 +84,10 @@ bool ConstantFPSDNode::isValueValidForType(MVT VT,
   
   // convert modifies in place, so make a copy.
   APFloat Val2 = APFloat(Val);
-  return Val2.convert(*MVTToAPFloatSemantics(VT),
-                      APFloat::rmNearestTiesToEven) == APFloat::opOK;
+  bool losesInfo;
+  (void) Val2.convert(*MVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
+                      &losesInfo);
+  return !losesInfo;
 }
 
 //===----------------------------------------------------------------------===//
@@ -95,7 +99,7 @@ bool ConstantFPSDNode::isValueValidForType(MVT VT,
 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
   // Look through a bit convert.
   if (N->getOpcode() == ISD::BIT_CONVERT)
-    N = N->getOperand(0).Val;
+    N = N->getOperand(0).getNode();
   
   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
   
@@ -116,7 +120,7 @@ bool ISD::isBuildVectorAllOnes(const SDNode *N) {
       return false;
   } else if (isa<ConstantFPSDNode>(NotZero)) {
     if (!cast<ConstantFPSDNode>(NotZero)->getValueAPF().
-                convertToAPInt().isAllOnesValue())
+                bitcastToAPInt().isAllOnesValue())
       return false;
   } else
     return false;
@@ -136,7 +140,7 @@ bool ISD::isBuildVectorAllOnes(const SDNode *N) {
 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
   // Look through a bit convert.
   if (N->getOpcode() == ISD::BIT_CONVERT)
-    N = N->getOperand(0).Val;
+    N = N->getOperand(0).getNode();
   
   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
   
@@ -212,7 +216,7 @@ ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
   unsigned OldG = (Operation >> 1) & 1;
   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
                        (OldL << 1) |       // New G bit
-                       (OldG << 2));        // New L bit.
+                       (OldG << 2));       // New L bit.
 }
 
 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
@@ -223,8 +227,10 @@ ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
     Operation ^= 7;   // Flip L, G, E bits, but not U.
   else
     Operation ^= 15;  // Flip all of the condition bits.
+
   if (Operation > ISD::SETTRUE2)
-    Operation &= ~8;     // Don't let N and U bits get set.
+    Operation &= ~8;  // Don't let N and U bits get set.
+
   return ISD::CondCode(Operation);
 }
 
@@ -301,7 +307,7 @@ ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
 }
 
 const TargetMachine &SelectionDAG::getTarget() const {
-  return TLI.getTargetMachine();
+  return MF->getTarget();
 }
 
 //===----------------------------------------------------------------------===//
@@ -325,8 +331,8 @@ static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
 static void AddNodeIDOperands(FoldingSetNodeID &ID,
                               const SDValue *Ops, unsigned NumOps) {
   for (; NumOps; --NumOps, ++Ops) {
-    ID.AddPointer(Ops->Val);
-    ID.AddInteger(Ops->ResNo);
+    ID.AddPointer(Ops->getNode());
+    ID.AddInteger(Ops->getResNo());
   }
 }
 
@@ -336,7 +342,7 @@ static void AddNodeIDOperands(FoldingSetNodeID &ID,
                               const SDUse *Ops, unsigned NumOps) {
   for (; NumOps; --NumOps, ++Ops) {
     ID.AddPointer(Ops->getVal());
-    ID.AddInteger(Ops->getSDValue().ResNo);
+    ID.AddInteger(Ops->getSDValue().getResNo());
   }
 }
 
@@ -348,17 +354,9 @@ static void AddNodeIDNode(FoldingSetNodeID &ID,
   AddNodeIDOperands(ID, OpList, N);
 }
 
-
-/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
-/// data.
-static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
-  AddNodeIDOpcode(ID, N->getOpcode());
-  // Add the return value info.
-  AddNodeIDValueTypes(ID, N->getVTList());
-  // Add the operand info.
-  AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands());
-
-  // Handle SDNode leafs with special info.
+/// AddNodeIDCustom - If this is an SDNode with special info, add this info to
+/// the NodeID data.
+static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
   switch (N->getOpcode()) {
   default: break;  // Normal nodes don't need extra info.
   case ISD::ARG_FLAGS:
@@ -366,18 +364,18 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
     break;
   case ISD::TargetConstant:
   case ISD::Constant:
-    ID.Add(cast<ConstantSDNode>(N)->getAPIntValue());
+    ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue());
     break;
   case ISD::TargetConstantFP:
   case ISD::ConstantFP: {
-    ID.Add(cast<ConstantFPSDNode>(N)->getValueAPF());
+    ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
     break;
   }
   case ISD::TargetGlobalAddress:
   case ISD::GlobalAddress:
   case ISD::TargetGlobalTLSAddress:
   case ISD::GlobalTLSAddress: {
-    GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
+    const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
     ID.AddPointer(GA->getGlobal());
     ID.AddInteger(GA->getOffset());
     break;
@@ -400,11 +398,7 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
     break;
   case ISD::MEMOPERAND: {
     const MachineMemOperand &MO = cast<MemOperandSDNode>(N)->MO;
-    ID.AddPointer(MO.getValue());
-    ID.AddInteger(MO.getFlags());
-    ID.AddInteger(MO.getOffset());
-    ID.AddInteger(MO.getSize());
-    ID.AddInteger(MO.getAlignment());
+    MO.Profile(ID);
     break;
   }
   case ISD::FrameIndex:
@@ -417,7 +411,7 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
     break;
   case ISD::ConstantPool:
   case ISD::TargetConstantPool: {
-    ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
+    const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
     ID.AddInteger(CP->getAlignment());
     ID.AddInteger(CP->getOffset());
     if (CP->isMachineConstantPoolEntry())
@@ -426,48 +420,131 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
       ID.AddPointer(CP->getConstVal());
     break;
   }
+  case ISD::CALL: {
+    const CallSDNode *Call = cast<CallSDNode>(N);
+    ID.AddInteger(Call->getCallingConv());
+    ID.AddInteger(Call->isVarArg());
+    break;
+  }
   case ISD::LOAD: {
-    LoadSDNode *LD = cast<LoadSDNode>(N);
+    const LoadSDNode *LD = cast<LoadSDNode>(N);
     ID.AddInteger(LD->getAddressingMode());
     ID.AddInteger(LD->getExtensionType());
     ID.AddInteger(LD->getMemoryVT().getRawBits());
-    ID.AddInteger(LD->getAlignment());
-    ID.AddInteger(LD->isVolatile());
+    ID.AddInteger(LD->getRawFlags());
     break;
   }
   case ISD::STORE: {
-    StoreSDNode *ST = cast<StoreSDNode>(N);
+    const StoreSDNode *ST = cast<StoreSDNode>(N);
     ID.AddInteger(ST->getAddressingMode());
     ID.AddInteger(ST->isTruncatingStore());
     ID.AddInteger(ST->getMemoryVT().getRawBits());
-    ID.AddInteger(ST->getAlignment());
-    ID.AddInteger(ST->isVolatile());
+    ID.AddInteger(ST->getRawFlags());
     break;
   }
-  case ISD::ATOMIC_CMP_SWAP:
-  case ISD::ATOMIC_LOAD_ADD:
-  case ISD::ATOMIC_SWAP:
-  case ISD::ATOMIC_LOAD_SUB:
-  case ISD::ATOMIC_LOAD_AND:
-  case ISD::ATOMIC_LOAD_OR:
-  case ISD::ATOMIC_LOAD_XOR:
-  case ISD::ATOMIC_LOAD_NAND:
-  case ISD::ATOMIC_LOAD_MIN:
-  case ISD::ATOMIC_LOAD_MAX:
-  case ISD::ATOMIC_LOAD_UMIN:
-  case ISD::ATOMIC_LOAD_UMAX: {
-    AtomicSDNode *AT = cast<AtomicSDNode>(N);
-    ID.AddInteger(AT->getAlignment());
-    ID.AddInteger(AT->isVolatile());
+  case ISD::ATOMIC_CMP_SWAP_8:
+  case ISD::ATOMIC_SWAP_8:
+  case ISD::ATOMIC_LOAD_ADD_8:
+  case ISD::ATOMIC_LOAD_SUB_8:
+  case ISD::ATOMIC_LOAD_AND_8:
+  case ISD::ATOMIC_LOAD_OR_8:
+  case ISD::ATOMIC_LOAD_XOR_8:
+  case ISD::ATOMIC_LOAD_NAND_8:
+  case ISD::ATOMIC_LOAD_MIN_8:
+  case ISD::ATOMIC_LOAD_MAX_8:
+  case ISD::ATOMIC_LOAD_UMIN_8:
+  case ISD::ATOMIC_LOAD_UMAX_8: 
+  case ISD::ATOMIC_CMP_SWAP_16:
+  case ISD::ATOMIC_SWAP_16:
+  case ISD::ATOMIC_LOAD_ADD_16:
+  case ISD::ATOMIC_LOAD_SUB_16:
+  case ISD::ATOMIC_LOAD_AND_16:
+  case ISD::ATOMIC_LOAD_OR_16:
+  case ISD::ATOMIC_LOAD_XOR_16:
+  case ISD::ATOMIC_LOAD_NAND_16:
+  case ISD::ATOMIC_LOAD_MIN_16:
+  case ISD::ATOMIC_LOAD_MAX_16:
+  case ISD::ATOMIC_LOAD_UMIN_16:
+  case ISD::ATOMIC_LOAD_UMAX_16: 
+  case ISD::ATOMIC_CMP_SWAP_32:
+  case ISD::ATOMIC_SWAP_32:
+  case ISD::ATOMIC_LOAD_ADD_32:
+  case ISD::ATOMIC_LOAD_SUB_32:
+  case ISD::ATOMIC_LOAD_AND_32:
+  case ISD::ATOMIC_LOAD_OR_32:
+  case ISD::ATOMIC_LOAD_XOR_32:
+  case ISD::ATOMIC_LOAD_NAND_32:
+  case ISD::ATOMIC_LOAD_MIN_32:
+  case ISD::ATOMIC_LOAD_MAX_32:
+  case ISD::ATOMIC_LOAD_UMIN_32:
+  case ISD::ATOMIC_LOAD_UMAX_32: 
+  case ISD::ATOMIC_CMP_SWAP_64:
+  case ISD::ATOMIC_SWAP_64:
+  case ISD::ATOMIC_LOAD_ADD_64:
+  case ISD::ATOMIC_LOAD_SUB_64:
+  case ISD::ATOMIC_LOAD_AND_64:
+  case ISD::ATOMIC_LOAD_OR_64:
+  case ISD::ATOMIC_LOAD_XOR_64:
+  case ISD::ATOMIC_LOAD_NAND_64:
+  case ISD::ATOMIC_LOAD_MIN_64:
+  case ISD::ATOMIC_LOAD_MAX_64:
+  case ISD::ATOMIC_LOAD_UMIN_64:
+  case ISD::ATOMIC_LOAD_UMAX_64: {
+    const AtomicSDNode *AT = cast<AtomicSDNode>(N);
+    ID.AddInteger(AT->getRawFlags());
     break;
   }
   } // end switch (N->getOpcode())
 }
 
+/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
+/// data.
+static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
+  AddNodeIDOpcode(ID, N->getOpcode());
+  // Add the return value info.
+  AddNodeIDValueTypes(ID, N->getVTList());
+  // Add the operand info.
+  AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands());
+
+  // Handle SDNode leafs with special info.
+  AddNodeIDCustom(ID, N);
+}
+
+/// encodeMemSDNodeFlags - Generic routine for computing a value for use in
+/// the CSE map that carries both alignment and volatility information.
+///
+static inline unsigned
+encodeMemSDNodeFlags(bool isVolatile, unsigned Alignment) {
+  return isVolatile | ((Log2_32(Alignment) + 1) << 1);
+}
+
 //===----------------------------------------------------------------------===//
 //                              SelectionDAG Class
 //===----------------------------------------------------------------------===//
 
+/// doNotCSE - Return true if CSE should not be performed for this node.
+static bool doNotCSE(SDNode *N) {
+  if (N->getValueType(0) == MVT::Flag)
+    return true; // Never CSE anything that produces a flag.
+
+  switch (N->getOpcode()) {
+  default: break;
+  case ISD::HANDLENODE:
+  case ISD::DBG_LABEL:
+  case ISD::DBG_STOPPOINT:
+  case ISD::EH_LABEL:
+  case ISD::DECLARE:
+    return true;   // Never CSE these nodes.
+  }
+
+  // Check that remaining values produced are not flags.
+  for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
+    if (N->getValueType(i) == MVT::Flag)
+      return true; // Never CSE anything that produces a flag.
+
+  return false;
+}
+
 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
 /// SelectionDAG.
 void SelectionDAG::RemoveDeadNodes() {
@@ -515,14 +592,15 @@ void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes,
       if (Operand->use_empty())
         DeadNodes.push_back(Operand);
     }
-    if (N->OperandsNeedDelete) {
+
+    if (N->OperandsNeedDelete)
       delete[] N->OperandList;
-    }
+
     N->OperandList = 0;
     N->NumOperands = 0;
     
     // Finally, remove N itself.
-    AllNodes.remove(N);
+    NodeAllocator.Deallocate(AllNodes.remove(N));
   }
 }
 
@@ -543,27 +621,30 @@ void SelectionDAG::DeleteNode(SDNode *N) {
 }
 
 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
-
-  // Drop all of the operands and decrement used nodes use counts.
+  // Drop all of the operands and decrement used node's use counts.
   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
     I->getVal()->removeUser(std::distance(N->op_begin(), I), N);
+
   if (N->OperandsNeedDelete) {
     delete[] N->OperandList;
+    N->OperandList = 0;
   }
-  N->OperandList = 0;
-  N->NumOperands = 0;
   
-  AllNodes.remove(N);
+  assert(N != AllNodes.begin());
+  NodeAllocator.Deallocate(AllNodes.remove(N));
 }
 
 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
 /// correspond to it.  This is useful when we're about to delete or repurpose
 /// the node.  We don't want future request for structurally identical nodes
 /// to return N anymore.
-void SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
+bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
   bool Erased = false;
   switch (N->getOpcode()) {
-  case ISD::HANDLENODE: return;  // noop.
+  case ISD::EntryToken:
+    assert(0 && "EntryToken should not be in CSEMaps!");
+    return false;
+  case ISD::HANDLENODE: return false;  // noop.
   case ISD::CONDCODE:
     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
            "Cond code doesn't exist!");
@@ -597,16 +678,13 @@ void SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
   // flag result (which cannot be CSE'd) or is one of the special cases that are
   // not subject to CSE.
   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Flag &&
-      !N->isTargetOpcode() &&
-      N->getOpcode() != ISD::DBG_LABEL &&
-      N->getOpcode() != ISD::DBG_STOPPOINT &&
-      N->getOpcode() != ISD::EH_LABEL &&
-      N->getOpcode() != ISD::DECLARE) {
+      !N->isMachineOpcode() && !doNotCSE(N)) {
     N->dump(this);
     cerr << "\n";
     assert(0 && "Node is not in map!");
   }
 #endif
+  return Erased;
 }
 
 /// AddNonLeafNodeToCSEMaps - Add the specified node back to the CSE maps.  It
@@ -617,24 +695,9 @@ void SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
 SDNode *SelectionDAG::AddNonLeafNodeToCSEMaps(SDNode *N) {
   assert(N->getNumOperands() && "This is a leaf node!");
 
-  if (N->getValueType(0) == MVT::Flag)
-    return 0;   // Never CSE anything that produces a flag.
+  if (doNotCSE(N))
+    return 0;
 
-  switch (N->getOpcode()) {
-  default: break;
-  case ISD::HANDLENODE:
-  case ISD::DBG_LABEL:
-  case ISD::DBG_STOPPOINT:
-  case ISD::EH_LABEL:
-  case ISD::DECLARE:
-    return 0;    // Never add these nodes.
-  }
-  
-  // Check that remaining values produced are not flags.
-  for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
-    if (N->getValueType(i) == MVT::Flag)
-      return 0;   // Never CSE anything that produces a flag.
-  
   SDNode *New = CSEMap.GetOrInsertNode(N);
   if (New != N) return New;  // Node already existed.
   return 0;
@@ -646,26 +709,13 @@ SDNode *SelectionDAG::AddNonLeafNodeToCSEMaps(SDNode *N) {
 /// node already exists with these operands, the slot will be non-null.
 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
                                            void *&InsertPos) {
-  if (N->getValueType(0) == MVT::Flag)
-    return 0;   // Never CSE anything that produces a flag.
+  if (doNotCSE(N))
+    return 0;
 
-  switch (N->getOpcode()) {
-  default: break;
-  case ISD::HANDLENODE:
-  case ISD::DBG_LABEL:
-  case ISD::DBG_STOPPOINT:
-  case ISD::EH_LABEL:
-    return 0;    // Never add these nodes.
-  }
-  
-  // Check that remaining values produced are not flags.
-  for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
-    if (N->getValueType(i) == MVT::Flag)
-      return 0;   // Never CSE anything that produces a flag.
-  
   SDValue Ops[] = { Op };
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1);
+  AddNodeIDCustom(ID, N);
   return CSEMap.FindNodeOrInsertPos(ID, InsertPos);
 }
 
@@ -676,16 +726,13 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 
                                            SDValue Op1, SDValue Op2,
                                            void *&InsertPos) {
-  if (N->getOpcode() == ISD::HANDLENODE || N->getValueType(0) == MVT::Flag)
-  
-  // Check that remaining values produced are not flags.
-  for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
-    if (N->getValueType(i) == MVT::Flag)
-      return 0;   // Never CSE anything that produces a flag.
-                                              
+  if (doNotCSE(N))
+    return 0;
+
   SDValue Ops[] = { Op1, Op2 };
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2);
+  AddNodeIDCustom(ID, N);
   return CSEMap.FindNodeOrInsertPos(ID, InsertPos);
 }
 
@@ -697,41 +744,12 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 
                                            const SDValue *Ops,unsigned NumOps,
                                            void *&InsertPos) {
-  if (N->getValueType(0) == MVT::Flag)
-    return 0;   // Never CSE anything that produces a flag.
+  if (doNotCSE(N))
+    return 0;
 
-  switch (N->getOpcode()) {
-  default: break;
-  case ISD::HANDLENODE:
-  case ISD::DBG_LABEL:
-  case ISD::DBG_STOPPOINT:
-  case ISD::EH_LABEL:
-  case ISD::DECLARE:
-    return 0;    // Never add these nodes.
-  }
-  
-  // Check that remaining values produced are not flags.
-  for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
-    if (N->getValueType(i) == MVT::Flag)
-      return 0;   // Never CSE anything that produces a flag.
-  
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps);
-  
-  if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
-    ID.AddInteger(LD->getAddressingMode());
-    ID.AddInteger(LD->getExtensionType());
-    ID.AddInteger(LD->getMemoryVT().getRawBits());
-    ID.AddInteger(LD->getAlignment());
-    ID.AddInteger(LD->isVolatile());
-  } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
-    ID.AddInteger(ST->getAddressingMode());
-    ID.AddInteger(ST->isTruncatingStore());
-    ID.AddInteger(ST->getMemoryVT().getRawBits());
-    ID.AddInteger(ST->getAlignment());
-    ID.AddInteger(ST->isVolatile());
-  }
-  
+  AddNodeIDCustom(ID, N);
   return CSEMap.FindNodeOrInsertPos(ID, InsertPos);
 }
 
@@ -740,15 +758,33 @@ void SelectionDAG::VerifyNode(SDNode *N) {
   switch (N->getOpcode()) {
   default:
     break;
+  case ISD::BUILD_PAIR: {
+    MVT VT = N->getValueType(0);
+    assert(N->getNumValues() == 1 && "Too many results!");
+    assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
+           "Wrong return type!");
+    assert(N->getNumOperands() == 2 && "Wrong number of operands!");
+    assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
+           "Mismatched operand types!");
+    assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
+           "Wrong operand type!");
+    assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
+           "Wrong return type size");
+    break;
+  }
   case ISD::BUILD_VECTOR: {
-    assert(N->getNumValues() == 1 && "Too many results for BUILD_VECTOR!");
-    assert(N->getValueType(0).isVector() && "Wrong BUILD_VECTOR return type!");
+    assert(N->getNumValues() == 1 && "Too many results!");
+    assert(N->getValueType(0).isVector() && "Wrong return type!");
     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
-           "Wrong number of BUILD_VECTOR operands!");
-    MVT EltVT = N->getValueType(0).getVectorElementType();
-    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
-      assert(I->getSDValue().getValueType() == EltVT &&
-             "Wrong BUILD_VECTOR operand type!");
+           "Wrong number of operands!");
+    // FIXME: Change vector_shuffle to a variadic node with mask elements being
+    // operands of the node.  Currently the mask is a BUILD_VECTOR passed as an
+    // operand, and it is not always possible to legalize it.  Turning off the
+    // following checks at least makes it possible to legalize most of the time.
+//    MVT EltVT = N->getValueType(0).getVectorElementType();
+//    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
+//      assert(I->getSDValue().getValueType() == EltVT &&
+//             "Wrong operand type!");
     break;
   }
   }
@@ -765,18 +801,56 @@ unsigned SelectionDAG::getMVTAlignment(MVT VT) const {
   return TLI.getTargetData()->getABITypeAlignment(Ty);
 }
 
+SelectionDAG::SelectionDAG(TargetLowering &tli, FunctionLoweringInfo &fli)
+  : TLI(tli), FLI(fli),
+    EntryNode(ISD::EntryToken, getVTList(MVT::Other)),
+    Root(getEntryNode()) {
+  AllNodes.push_back(&EntryNode);
+}
+
+void SelectionDAG::init(MachineFunction &mf, MachineModuleInfo *mmi) {
+  MF = &mf;
+  MMI = mmi;
+}
+
 SelectionDAG::~SelectionDAG() {
+  allnodes_clear();
+}
+
+void SelectionDAG::allnodes_clear() {
+  assert(&*AllNodes.begin() == &EntryNode);
+  AllNodes.remove(AllNodes.begin());
   while (!AllNodes.empty()) {
     SDNode *N = AllNodes.remove(AllNodes.begin());
     N->SetNextInBucket(0);
+
     if (N->OperandsNeedDelete) {
       delete [] N->OperandList;
+      N->OperandList = 0;
     }
-    N->OperandList = 0;
-    N->NumOperands = 0;
+
+    NodeAllocator.Deallocate(N);
   }
 }
 
+void SelectionDAG::clear() {
+  allnodes_clear();
+  OperandAllocator.Reset();
+  CSEMap.clear();
+
+  ExtendedValueTypeNodes.clear();
+  ExternalSymbols.clear();
+  TargetExternalSymbols.clear();
+  std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
+            static_cast<CondCodeSDNode*>(0));
+  std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
+            static_cast<SDNode*>(0));
+
+  EntryNode.Uses = 0;
+  AllNodes.push_back(&EntryNode);
+  Root = getEntryNode();
+}
+
 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, MVT VT) {
   if (Op.getValueType() == VT) return Op;
   APInt Imm = APInt::getLowBitsSet(Op.getValueSizeInBits(),
@@ -791,6 +865,10 @@ SDValue SelectionDAG::getConstant(uint64_t Val, MVT VT, bool isT) {
 }
 
 SDValue SelectionDAG::getConstant(const APInt &Val, MVT VT, bool isT) {
+  return getConstant(*ConstantInt::get(Val), VT, isT);
+}
+
+SDValue SelectionDAG::getConstant(const ConstantInt &Val, MVT VT, bool isT) {
   assert(VT.isInteger() && "Cannot create FP integer constant!");
 
   MVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT;
@@ -800,7 +878,7 @@ SDValue SelectionDAG::getConstant(const APInt &Val, MVT VT, bool isT) {
   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
-  ID.Add(Val);
+  ID.AddPointer(&Val);
   void *IP = 0;
   SDNode *N = NULL;
   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
@@ -808,7 +886,7 @@ SDValue SelectionDAG::getConstant(const APInt &Val, MVT VT, bool isT) {
       return SDValue(N, 0);
   if (!N) {
     N = NodeAllocator.Allocate<ConstantSDNode>();
-    new (N) ConstantSDNode(isT, Val, EltVT);
+    new (N) ConstantSDNode(isT, &Val, EltVT);
     CSEMap.InsertNode(N, IP);
     AllNodes.push_back(N);
   }
@@ -828,6 +906,10 @@ SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
 
 
 SDValue SelectionDAG::getConstantFP(const APFloat& V, MVT VT, bool isTarget) {
+  return getConstantFP(*ConstantFP::get(V), VT, isTarget);
+}
+
+SDValue SelectionDAG::getConstantFP(const ConstantFP& V, MVT VT, bool isTarget){
   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
                                 
   MVT EltVT =
@@ -839,7 +921,7 @@ SDValue SelectionDAG::getConstantFP(const APFloat& V, MVT VT, bool isTarget) {
   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
-  ID.Add(V);
+  ID.AddPointer(&V);
   void *IP = 0;
   SDNode *N = NULL;
   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
@@ -847,7 +929,7 @@ SDValue SelectionDAG::getConstantFP(const APFloat& V, MVT VT, bool isTarget) {
       return SDValue(N, 0);
   if (!N) {
     N = NodeAllocator.Allocate<ConstantFPSDNode>();
-    new (N) ConstantFPSDNode(isTarget, V, EltVT);
+    new (N) ConstantFPSDNode(isTarget, &V, EltVT);
     CSEMap.InsertNode(N, IP);
     AllNodes.push_back(N);
   }
@@ -871,15 +953,20 @@ SDValue SelectionDAG::getConstantFP(double Val, MVT VT, bool isTarget) {
 }
 
 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV,
-                                       MVT VT, int Offset,
+                                       MVT VT, int64_t Offset,
                                        bool isTargetGA) {
   unsigned Opc;
 
+  // Truncate (with sign-extension) the offset value to the pointer size.
+  unsigned BitWidth = TLI.getPointerTy().getSizeInBits();
+  if (BitWidth < 64)
+    Offset = (Offset << (64 - BitWidth) >> (64 - BitWidth));
+
   const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
   if (!GVar) {
     // If GV is an alias then use the aliasee for determining thread-localness.
     if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
-      GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal());
+      GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false));
   }
 
   if (GVar && GVar->isThreadLocal())
@@ -934,6 +1021,9 @@ SDValue SelectionDAG::getJumpTable(int JTI, MVT VT, bool isTarget){
 SDValue SelectionDAG::getConstantPool(Constant *C, MVT VT,
                                       unsigned Alignment, int Offset,
                                       bool isTarget) {
+  if (Alignment == 0)
+    Alignment =
+      TLI.getTargetData()->getPreferredTypeAlignmentShift(C->getType());
   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
@@ -954,6 +1044,9 @@ SDValue SelectionDAG::getConstantPool(Constant *C, MVT VT,
 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, MVT VT,
                                       unsigned Alignment, int Offset,
                                       bool isTarget) {
+  if (Alignment == 0)
+    Alignment =
+      TLI.getTargetData()->getPreferredTypeAlignmentShift(C->getType());
   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
@@ -1044,6 +1137,21 @@ SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
   return SDValue(CondCodeNodes[Cond], 0);
 }
 
+SDValue SelectionDAG::getConvertRndSat(MVT VT, SDValue Val, SDValue DTy,
+                                       SDValue STy, SDValue Rnd, SDValue Sat,
+                                       ISD::CvtCode Code) {
+  FoldingSetNodeID ID;
+  void* IP = 0;
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    return SDValue(E, 0);
+  CvtRndSatSDNode *N = NodeAllocator.Allocate<CvtRndSatSDNode>();
+  SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
+  new (N) CvtRndSatSDNode(VT, Ops, 5, Code);
+  CSEMap.InsertNode(N, IP);
+  AllNodes.push_back(N);
+  return SDValue(N, 0);
+}
+
 SDValue SelectionDAG::getRegister(unsigned RegNo, MVT VT) {
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0);
@@ -1104,17 +1212,15 @@ SDValue SelectionDAG::getSrcValue(const Value *V) {
 }
 
 SDValue SelectionDAG::getMemOperand(const MachineMemOperand &MO) {
+#ifndef NDEBUG
   const Value *v = MO.getValue();
   assert((!v || isa<PointerType>(v->getType())) &&
          "SrcValue is not a pointer?");
+#endif
 
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, ISD::MEMOPERAND, getVTList(MVT::Other), 0, 0);
-  ID.AddPointer(v);
-  ID.AddInteger(MO.getFlags());
-  ID.AddInteger(MO.getOffset());
-  ID.AddInteger(MO.getSize());
-  ID.AddInteger(MO.getAlignment());
+  MO.Profile(ID);
 
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
@@ -1131,7 +1237,7 @@ SDValue SelectionDAG::getMemOperand(const MachineMemOperand &MO) {
 /// specified value type.
 SDValue SelectionDAG::CreateStackTemporary(MVT VT, unsigned minAlign) {
   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
-  unsigned ByteSize = VT.getSizeInBits()/8;
+  unsigned ByteSize = VT.getStoreSizeInBits()/8;
   const Type *Ty = VT.getTypeForMVT();
   unsigned StackAlign =
   std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign);
@@ -1140,6 +1246,22 @@ SDValue SelectionDAG::CreateStackTemporary(MVT VT, unsigned minAlign) {
   return getFrameIndex(FrameIdx, TLI.getPointerTy());
 }
 
+/// CreateStackTemporary - Create a stack temporary suitable for holding
+/// either of the specified value types.
+SDValue SelectionDAG::CreateStackTemporary(MVT VT1, MVT VT2) {
+  unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
+                            VT2.getStoreSizeInBits())/8;
+  const Type *Ty1 = VT1.getTypeForMVT();
+  const Type *Ty2 = VT2.getTypeForMVT();
+  const TargetData *TD = TLI.getTargetData();
+  unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
+                            TD->getPrefTypeAlignment(Ty2));
+
+  MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
+  int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align);
+  return getFrameIndex(FrameIdx, TLI.getPointerTy());
+}
+
 SDValue SelectionDAG::FoldSetCC(MVT VT, SDValue N1,
                                 SDValue N2, ISD::CondCode Cond) {
   // These setcc operations always fold.
@@ -1164,9 +1286,9 @@ SDValue SelectionDAG::FoldSetCC(MVT VT, SDValue N1,
     break;
   }
   
-  if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val)) {
+  if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
     const APInt &C2 = N2C->getAPIntValue();
-    if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val)) {
+    if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
       const APInt &C1 = N1C->getAPIntValue();
       
       switch (Cond) {
@@ -1184,8 +1306,8 @@ SDValue SelectionDAG::FoldSetCC(MVT VT, SDValue N1,
       }
     }
   }
-  if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.Val)) {
-    if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.Val)) {
+  if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
+    if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
       // No compile time operations on this type yet.
       if (N1C->getValueType(0) == MVT::ppcf128)
         return SDValue();
@@ -1387,16 +1509,21 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask,
     KnownOne &= KnownOne2;
     KnownZero &= KnownZero2;
     return;
+  case ISD::SADDO:
+  case ISD::UADDO:
+    if (Op.getResNo() != 1)
+      return;
+    // The boolean result conforms to getBooleanContents.  Fall through.
   case ISD::SETCC:
     // If we know the result of a setcc has the top bits zero, use this info.
-    if (TLI.getSetCCResultContents() == TargetLowering::ZeroOrOneSetCCResult &&
+    if (TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent &&
         BitWidth > 1)
       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
     return;
   case ISD::SHL:
     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
-      unsigned ShAmt = SA->getValue();
+      unsigned ShAmt = SA->getZExtValue();
 
       // If the shift count is an invalid immediate, don't do anything.
       if (ShAmt >= BitWidth)
@@ -1414,7 +1541,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask,
   case ISD::SRL:
     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
-      unsigned ShAmt = SA->getValue();
+      unsigned ShAmt = SA->getZExtValue();
 
       // If the shift count is an invalid immediate, don't do anything.
       if (ShAmt >= BitWidth)
@@ -1432,7 +1559,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask,
     return;
   case ISD::SRA:
     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
-      unsigned ShAmt = SA->getValue();
+      unsigned ShAmt = SA->getZExtValue();
 
       // If the shift count is an invalid immediate, don't do anything.
       if (ShAmt >= BitWidth)
@@ -1506,7 +1633,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask,
     return;
   }
   case ISD::LOAD: {
-    if (ISD::isZEXTLoad(Op.Val)) {
+    if (ISD::isZEXTLoad(Op.getNode())) {
       LoadSDNode *LD = cast<LoadSDNode>(Op);
       MVT VT = LD->getMemoryVT();
       unsigned MemBits = VT.getSizeInBits();
@@ -1656,15 +1783,13 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask,
         APInt Mask2 = LowBits | APInt::getSignBit(BitWidth);
         ComputeMaskedBits(Op.getOperand(0), Mask2,KnownZero2,KnownOne2,Depth+1);
 
-        // The sign of a remainder is equal to the sign of the first
-        // operand (zero being positive).
+        // If the sign bit of the first operand is zero, the sign bit of
+        // the result is zero. If the first operand has no one bits below
+        // the second operand's single 1 bit, its sign will be zero.
         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
           KnownZero2 |= ~LowBits;
-        else if (KnownOne2[BitWidth-1])
-          KnownOne2 |= ~LowBits;
 
         KnownZero |= KnownZero2 & Mask;
-        KnownOne |= KnownOne2 & Mask;
 
         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
       }
@@ -1759,7 +1884,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
     // SRA X, C   -> adds C sign bits.
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
-      Tmp += C->getValue();
+      Tmp += C->getZExtValue();
       if (Tmp > VTBits) Tmp = VTBits;
     }
     return Tmp;
@@ -1767,9 +1892,9 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
       // shl destroys sign bits.
       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
-      if (C->getValue() >= VTBits ||      // Bad shift.
-          C->getValue() >= Tmp) break;    // Shifted all sign bits out.
-      return Tmp - C->getValue();
+      if (C->getZExtValue() >= VTBits ||      // Bad shift.
+          C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
+      return Tmp - C->getZExtValue();
     }
     break;
   case ISD::AND:
@@ -1791,17 +1916,22 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
     if (Tmp == 1) return 1;  // Early out.
     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
     return std::min(Tmp, Tmp2);
-    
+
+  case ISD::SADDO:
+  case ISD::UADDO:
+    if (Op.getResNo() != 1)
+      break;
+    // The boolean result conforms to getBooleanContents.  Fall through.
   case ISD::SETCC:
     // If setcc returns 0/-1, all bits are sign bits.
-    if (TLI.getSetCCResultContents() ==
-        TargetLowering::ZeroOrNegativeOneSetCCResult)
+    if (TLI.getBooleanContents() ==
+        TargetLowering::ZeroOrNegativeOneBooleanContent)
       return VTBits;
     break;
   case ISD::ROTL:
   case ISD::ROTR:
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
-      unsigned RotAmt = C->getValue() & (VTBits-1);
+      unsigned RotAmt = C->getZExtValue() & (VTBits-1);
       
       // Handle rotate right by N like a rotate left by 32-N.
       if (Op.getOpcode() == ISD::ROTR)
@@ -1929,6 +2059,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
 bool SelectionDAG::isVerifiedDebugInfoDesc(SDValue Op) const {
   GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op);
   if (!GA) return false;
+  if (GA->getOffset() != 0) return false;
   GlobalVariable *GV = dyn_cast<GlobalVariable>(GA->getGlobal());
   if (!GV) return false;
   MachineModuleInfo *MMI = getMachineModuleInfo();
@@ -1944,14 +2075,15 @@ SDValue SelectionDAG::getShuffleScalarElt(const SDNode *N, unsigned i) {
   SDValue Idx = PermMask.getOperand(i);
   if (Idx.getOpcode() == ISD::UNDEF)
     return getNode(ISD::UNDEF, VT.getVectorElementType());
-  unsigned Index = cast<ConstantSDNode>(Idx)->getValue();
+  unsigned Index = cast<ConstantSDNode>(Idx)->getZExtValue();
   unsigned NumElems = PermMask.getNumOperands();
   SDValue V = (Index < NumElems) ? N->getOperand(0) : N->getOperand(1);
   Index %= NumElems;
 
   if (V.getOpcode() == ISD::BIT_CONVERT) {
     V = V.getOperand(0);
-    if (V.getValueType().getVectorNumElements() != NumElems)
+    MVT VVT = V.getValueType();
+    if (!VVT.isVector() || VVT.getVectorNumElements() != NumElems)
       return SDValue();
   }
   if (V.getOpcode() == ISD::SCALAR_TO_VECTOR)
@@ -1960,7 +2092,7 @@ SDValue SelectionDAG::getShuffleScalarElt(const SDNode *N, unsigned i) {
   if (V.getOpcode() == ISD::BUILD_VECTOR)
     return V.getOperand(Index);
   if (V.getOpcode() == ISD::VECTOR_SHUFFLE)
-    return getShuffleScalarElt(V.Val, Index);
+    return getShuffleScalarElt(V.getNode(), Index);
   return SDValue();
 }
 
@@ -1986,7 +2118,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT) {
 
 SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
   // Constant fold unary operations with an integer constant operand.
-  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.Val)) {
+  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
     const APInt &Val = C->getAPIntValue();
     unsigned BitWidth = VT.getSizeInBits();
     switch (Opcode) {
@@ -2027,7 +2159,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
   }
 
   // Constant fold unary operations with a floating point constant operand.
-  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.Val)) {
+  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
     APFloat V = C->getValueAPF();    // make copy
     if (VT != MVT::ppcf128 && Operand.getValueType() != MVT::ppcf128) {
       switch (Opcode) {
@@ -2038,38 +2170,43 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
         V.clearSign();
         return getConstantFP(V, VT);
       case ISD::FP_ROUND:
-      case ISD::FP_EXTEND:
+      case ISD::FP_EXTEND: {
+        bool ignored;
         // This can return overflow, underflow, or inexact; we don't care.
         // FIXME need to be more flexible about rounding mode.
         (void)V.convert(*MVTToAPFloatSemantics(VT),
-                        APFloat::rmNearestTiesToEven);
+                        APFloat::rmNearestTiesToEven, &ignored);
         return getConstantFP(V, VT);
+      }
       case ISD::FP_TO_SINT:
       case ISD::FP_TO_UINT: {
         integerPart x;
+        bool ignored;
         assert(integerPartWidth >= 64);
         // FIXME need to be more flexible about rounding mode.
         APFloat::opStatus s = V.convertToInteger(&x, 64U,
                               Opcode==ISD::FP_TO_SINT,
-                              APFloat::rmTowardZero);
+                              APFloat::rmTowardZero, &ignored);
         if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
           break;
         return getConstant(x, VT);
       }
       case ISD::BIT_CONVERT:
         if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
-          return getConstant((uint32_t)V.convertToAPInt().getZExtValue(), VT);
+          return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
         else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
-          return getConstant(V.convertToAPInt().getZExtValue(), VT);
+          return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
         break;
       }
     }
   }
 
-  unsigned OpOpcode = Operand.Val->getOpcode();
+  unsigned OpOpcode = Operand.getNode()->getOpcode();
   switch (Opcode) {
   case ISD::TokenFactor:
-    return Operand;         // Factor of one node?  No need.
+  case ISD::MERGE_VALUES:
+  case ISD::CONCAT_VECTORS:
+    return Operand;         // Factor, merge or concat of one node?  No need.
   case ISD::FP_ROUND: assert(0 && "Invalid method to make FP_ROUND node");
   case ISD::FP_EXTEND:
     assert(VT.isFloatingPoint() &&
@@ -2085,7 +2222,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
     assert(Operand.getValueType().bitsLT(VT)
            && "Invalid sext node, dst < src!");
     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
-      return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
+      return getNode(OpOpcode, VT, Operand.getNode()->getOperand(0));
     break;
   case ISD::ZERO_EXTEND:
     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
@@ -2094,7 +2231,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
     assert(Operand.getValueType().bitsLT(VT)
            && "Invalid zext node, dst < src!");
     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
-      return getNode(ISD::ZERO_EXTEND, VT, Operand.Val->getOperand(0));
+      return getNode(ISD::ZERO_EXTEND, VT, Operand.getNode()->getOperand(0));
     break;
   case ISD::ANY_EXTEND:
     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
@@ -2104,7 +2241,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
            && "Invalid anyext node, dst < src!");
     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND)
       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
-      return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
+      return getNode(OpOpcode, VT, Operand.getNode()->getOperand(0));
     break;
   case ISD::TRUNCATE:
     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
@@ -2113,16 +2250,16 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
     assert(Operand.getValueType().bitsGT(VT)
            && "Invalid truncate node, src < dst!");
     if (OpOpcode == ISD::TRUNCATE)
-      return getNode(ISD::TRUNCATE, VT, Operand.Val->getOperand(0));
+      return getNode(ISD::TRUNCATE, VT, Operand.getNode()->getOperand(0));
     else if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
              OpOpcode == ISD::ANY_EXTEND) {
       // If the source is smaller than the dest, we still need an extend.
-      if (Operand.Val->getOperand(0).getValueType().bitsLT(VT))
-        return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
-      else if (Operand.Val->getOperand(0).getValueType().bitsGT(VT))
-        return getNode(ISD::TRUNCATE, VT, Operand.Val->getOperand(0));
+      if (Operand.getNode()->getOperand(0).getValueType().bitsLT(VT))
+        return getNode(OpOpcode, VT, Operand.getNode()->getOperand(0));
+      else if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
+        return getNode(ISD::TRUNCATE, VT, Operand.getNode()->getOperand(0));
       else
-        return Operand.Val->getOperand(0);
+        return Operand.getNode()->getOperand(0);
     }
     break;
   case ISD::BIT_CONVERT:
@@ -2150,14 +2287,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
     break;
   case ISD::FNEG:
     if (OpOpcode == ISD::FSUB)   // -(X-Y) -> (Y-X)
-      return getNode(ISD::FSUB, VT, Operand.Val->getOperand(1),
-                     Operand.Val->getOperand(0));
+      return getNode(ISD::FSUB, VT, Operand.getNode()->getOperand(1),
+                     Operand.getNode()->getOperand(0));
     if (OpOpcode == ISD::FNEG)  // --X -> X
-      return Operand.Val->getOperand(0);
+      return Operand.getNode()->getOperand(0);
     break;
   case ISD::FABS:
     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
-      return getNode(ISD::FABS, VT, Operand.Val->getOperand(0));
+      return getNode(ISD::FABS, VT, Operand.getNode()->getOperand(0));
     break;
   }
 
@@ -2185,10 +2322,46 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
   return SDValue(N, 0);
 }
 
+SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode,
+                                             MVT VT,
+                                             ConstantSDNode *Cst1,
+                                             ConstantSDNode *Cst2) {
+  const APInt &C1 = Cst1->getAPIntValue(), &C2 = Cst2->getAPIntValue();
+
+  switch (Opcode) {
+  case ISD::ADD:  return getConstant(C1 + C2, VT);
+  case ISD::SUB:  return getConstant(C1 - C2, VT);
+  case ISD::MUL:  return getConstant(C1 * C2, VT);
+  case ISD::UDIV:
+    if (C2.getBoolValue()) return getConstant(C1.udiv(C2), VT);
+    break;
+  case ISD::UREM:
+    if (C2.getBoolValue()) return getConstant(C1.urem(C2), VT);
+    break;
+  case ISD::SDIV:
+    if (C2.getBoolValue()) return getConstant(C1.sdiv(C2), VT);
+    break;
+  case ISD::SREM:
+    if (C2.getBoolValue()) return getConstant(C1.srem(C2), VT);
+    break;
+  case ISD::AND:  return getConstant(C1 & C2, VT);
+  case ISD::OR:   return getConstant(C1 | C2, VT);
+  case ISD::XOR:  return getConstant(C1 ^ C2, VT);
+  case ISD::SHL:  return getConstant(C1 << C2, VT);
+  case ISD::SRL:  return getConstant(C1.lshr(C2), VT);
+  case ISD::SRA:  return getConstant(C1.ashr(C2), VT);
+  case ISD::ROTL: return getConstant(C1.rotl(C2), VT);
+  case ISD::ROTR: return getConstant(C1.rotr(C2), VT);
+  default: break;
+  }
+
+  return SDValue();
+}
+
 SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
                               SDValue N1, SDValue N2) {
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
-  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val);
+  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
+  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
   switch (Opcode) {
   default: break;
   case ISD::TokenFactor:
@@ -2197,6 +2370,17 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
     // Fold trivial token factors.
     if (N1.getOpcode() == ISD::EntryToken) return N2;
     if (N2.getOpcode() == ISD::EntryToken) return N1;
+    if (N1 == N2) return N1;
+    break;
+  case ISD::CONCAT_VECTORS:
+    // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
+    // one big BUILD_VECTOR.
+    if (N1.getOpcode() == ISD::BUILD_VECTOR &&
+        N2.getOpcode() == ISD::BUILD_VECTOR) {
+      SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), N1.getNode()->op_end());
+      Elts.insert(Elts.end(), N2.getNode()->op_begin(), N2.getNode()->op_end());
+      return getNode(ISD::BUILD_VECTOR, VT, &Elts[0], Elts.size());
+    }
     break;
   case ISD::AND:
     assert(VT.isInteger() && N1.getValueType() == N2.getValueType() &&
@@ -2251,6 +2435,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
            "Shift operators return type must be the same as their first arg");
     assert(VT.isInteger() && N2.getValueType().isInteger() &&
            "Shifts only work on integers");
+    assert((N2.getValueType() == TLI.getShiftAmountTy() ||
+            (N2.getValueType().isVector() && N2.getValueType().isInteger())) &&
+           "Wrong type for shift amount");
 
     // Always fold shifts of i1 values so the code generator doesn't need to
     // handle them.  Since we know the size of the shift has to be less than the
@@ -2302,40 +2489,39 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
     break;
   }
   case ISD::EXTRACT_VECTOR_ELT:
-    assert(N2C && "Bad EXTRACT_VECTOR_ELT!");
-
     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
     if (N1.getOpcode() == ISD::UNDEF)
       return getNode(ISD::UNDEF, VT);
       
     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
     // expanding copies of large vectors from registers.
-    if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
+    if (N2C &&
+        N1.getOpcode() == ISD::CONCAT_VECTORS &&
         N1.getNumOperands() > 0) {
       unsigned Factor =
         N1.getOperand(0).getValueType().getVectorNumElements();
       return getNode(ISD::EXTRACT_VECTOR_ELT, VT,
-                     N1.getOperand(N2C->getValue() / Factor),
-                     getConstant(N2C->getValue() % Factor, N2.getValueType()));
+                     N1.getOperand(N2C->getZExtValue() / Factor),
+                     getConstant(N2C->getZExtValue() % Factor,
+                                 N2.getValueType()));
     }
 
     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
     // expanding large vector constants.
-    if (N1.getOpcode() == ISD::BUILD_VECTOR)
-      return N1.getOperand(N2C->getValue());
+    if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR)
+      return N1.getOperand(N2C->getZExtValue());
       
     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
     // operations are lowered to scalars.
-    if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT)
-      if (ConstantSDNode *IEC = dyn_cast<ConstantSDNode>(N1.getOperand(2))) {
-        if (IEC == N2C)
-          return N1.getOperand(1);
-        else
-          return getNode(ISD::EXTRACT_VECTOR_ELT, VT, N1.getOperand(0), N2);
-      }
+    if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
+      if (N1.getOperand(2) == N2)
+        return N1.getOperand(1);
+      else
+        return getNode(ISD::EXTRACT_VECTOR_ELT, VT, N1.getOperand(0), N2);
+    }
     break;
   case ISD::EXTRACT_ELEMENT:
-    assert(N2C && (unsigned)N2C->getValue() < 2 && "Bad EXTRACT_ELEMENT!");
+    assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
     assert(!N1.getValueType().isVector() && !VT.isVector() &&
            (N1.getValueType().isInteger() == VT.isInteger()) &&
            "Wrong types for EXTRACT_ELEMENT!");
@@ -2344,12 +2530,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
     // 64-bit integers into 32-bit parts.  Instead of building the extract of
     // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 
     if (N1.getOpcode() == ISD::BUILD_PAIR)
-      return N1.getOperand(N2C->getValue());
+      return N1.getOperand(N2C->getZExtValue());
 
     // EXTRACT_ELEMENT of a constant int is also very common.
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
       unsigned ElementSize = VT.getSizeInBits();
-      unsigned Shift = ElementSize * N2C->getValue();
+      unsigned Shift = ElementSize * N2C->getZExtValue();
       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
       return getConstant(ShiftedVal.trunc(ElementSize), VT);
     }
@@ -2362,33 +2548,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
 
   if (N1C) {
     if (N2C) {
-      const APInt &C1 = N1C->getAPIntValue(), &C2 = N2C->getAPIntValue();
-      switch (Opcode) {
-      case ISD::ADD: return getConstant(C1 + C2, VT);
-      case ISD::SUB: return getConstant(C1 - C2, VT);
-      case ISD::MUL: return getConstant(C1 * C2, VT);
-      case ISD::UDIV:
-        if (C2.getBoolValue()) return getConstant(C1.udiv(C2), VT);
-        break;
-      case ISD::UREM :
-        if (C2.getBoolValue()) return getConstant(C1.urem(C2), VT);
-        break;
-      case ISD::SDIV :
-        if (C2.getBoolValue()) return getConstant(C1.sdiv(C2), VT);
-        break;
-      case ISD::SREM :
-        if (C2.getBoolValue()) return getConstant(C1.srem(C2), VT);
-        break;
-      case ISD::AND  : return getConstant(C1 & C2, VT);
-      case ISD::OR   : return getConstant(C1 | C2, VT);
-      case ISD::XOR  : return getConstant(C1 ^ C2, VT);
-      case ISD::SHL  : return getConstant(C1 << C2, VT);
-      case ISD::SRL  : return getConstant(C1.lshr(C2), VT);
-      case ISD::SRA  : return getConstant(C1.ashr(C2), VT);
-      case ISD::ROTL : return getConstant(C1.rotl(C2), VT);
-      case ISD::ROTR : return getConstant(C1.rotr(C2), VT);
-      default: break;
-      }
+      SDValue SV = FoldConstantArithmetic(Opcode, VT, N1C, N2C);
+      if (SV.getNode()) return SV;
     } else {      // Cannonicalize constant to RHS if commutative
       if (isCommutativeBinOp(Opcode)) {
         std::swap(N1C, N2C);
@@ -2398,8 +2559,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
   }
 
   // Constant fold FP operations.
-  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.Val);
-  ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.Val);
+  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
+  ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
   if (N1CFP) {
     if (!N2CFP && isCommutativeBinOp(Opcode)) {
       // Cannonicalize constant to RHS if commutative
@@ -2542,18 +2703,30 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
 SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
                               SDValue N1, SDValue N2, SDValue N3) {
   // Perform various simplifications.
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
-  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val);
+  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
+  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
   switch (Opcode) {
+  case ISD::CONCAT_VECTORS:
+    // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
+    // one big BUILD_VECTOR.
+    if (N1.getOpcode() == ISD::BUILD_VECTOR &&
+        N2.getOpcode() == ISD::BUILD_VECTOR &&
+        N3.getOpcode() == ISD::BUILD_VECTOR) {
+      SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), N1.getNode()->op_end());
+      Elts.insert(Elts.end(), N2.getNode()->op_begin(), N2.getNode()->op_end());
+      Elts.insert(Elts.end(), N3.getNode()->op_begin(), N3.getNode()->op_end());
+      return getNode(ISD::BUILD_VECTOR, VT, &Elts[0], Elts.size());
+    }
+    break;
   case ISD::SETCC: {
     // Use FoldSetCC to simplify SETCC's.
     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get());
-    if (Simp.Val) return Simp;
+    if (Simp.getNode()) return Simp;
     break;
   }
   case ISD::SELECT:
     if (N1C) {
-     if (N1C->getValue())
+     if (N1C->getZExtValue())
         return N2;             // select true, X, Y -> X
       else
         return N3;             // select false, X, Y -> Y
@@ -2563,14 +2736,15 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
     break;
   case ISD::BRCOND:
     if (N2C) {
-      if (N2C->getValue()) // Unconditional branch
+      if (N2C->getZExtValue()) // Unconditional branch
         return getNode(ISD::BR, MVT::Other, N1, N3);
       else
         return N1;         // Never-taken branch
     }
     break;
   case ISD::VECTOR_SHUFFLE:
-    assert(VT == N1.getValueType() && VT == N2.getValueType() &&
+    assert(N1.getValueType() == N2.getValueType() &&
+           N1.getValueType().isVector() &&
            VT.isVector() && N3.getValueType().isVector() &&
            N3.getOpcode() == ISD::BUILD_VECTOR &&
            VT.getVectorNumElements() == N3.getNumOperands() &&
@@ -2627,7 +2801,7 @@ static SDValue getMemsetValue(SDValue Value, MVT VT, SelectionDAG &DAG) {
   unsigned NumBits = VT.isVector() ?
     VT.getVectorElementType().getSizeInBits() : VT.getSizeInBits();
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
-    APInt Val = APInt(NumBits, C->getValue() & 255);
+    APInt Val = APInt(NumBits, C->getZExtValue() & 255);
     unsigned Shift = 8;
     for (unsigned i = NumBits; i > 8; i >>= 1) {
       Val = (Val << Shift) | Val;
@@ -2638,12 +2812,15 @@ static SDValue getMemsetValue(SDValue Value, MVT VT, SelectionDAG &DAG) {
     return DAG.getConstantFP(APFloat(Val), VT);
   }
 
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
   unsigned Shift = 8;
   for (unsigned i = NumBits; i > 8; i >>= 1) {
     Value = DAG.getNode(ISD::OR, VT,
                         DAG.getNode(ISD::SHL, VT, Value,
-                                    DAG.getConstant(Shift, MVT::i8)), Value);
+                                    DAG.getConstant(Shift,
+                                                    TLI.getShiftAmountTy())),
+                        Value);
     Shift <<= 1;
   }
 
@@ -2698,7 +2875,7 @@ static bool isMemSrcFromString(SDValue Src, std::string &Str) {
            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
            Src.getOperand(1).getOpcode() == ISD::Constant) {
     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
-    SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getValue();
+    SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
   }
   if (!G)
     return false;
@@ -2723,7 +2900,7 @@ bool MeetsMaxMemopRequirement(std::vector<MVT> &MemOps,
   isSrcStr = isMemSrcFromString(Src, Str);
   bool isSrcConst = isa<ConstantSDNode>(Src);
   bool AllowUnalign = TLI.allowsUnalignedMemoryAccesses();
-  MVT VT= TLI.getOptimalMemOpType(Size, Align, isSrcConst, isSrcStr);
+  MVT VT = TLI.getOptimalMemOpType(Size, Align, isSrcConst, isSrcStr);
   if (VT != MVT::iAny) {
     unsigned NewAlign = (unsigned)
       TLI.getTargetData()->getABITypeAlignment(VT.getTypeForMVT());
@@ -2807,7 +2984,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG,
   // Expand memcpy to a series of load and store ops if the size operand falls
   // below a certain threshold.
   std::vector<MVT> MemOps;
-  uint64_t Limit = -1;
+  uint64_t Limit = -1ULL;
   if (!AlwaysInline)
     Limit = TLI.getMaxStoresPerMemcpy();
   unsigned DstAlign = Align;  // Destination alignment can change.
@@ -2865,7 +3042,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG,
   // Expand memmove to a series of load and store ops if the size operand falls
   // below a certain threshold.
   std::vector<MVT> MemOps;
-  uint64_t Limit = -1;
+  uint64_t Limit = -1ULL;
   if (!AlwaysInline)
     Limit = TLI.getMaxStoresPerMemmove();
   unsigned DstAlign = Align;  // Destination alignment can change.
@@ -2937,8 +3114,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG,
     unsigned VTSize = VT.getSizeInBits() / 8;
     SDValue Value = getMemsetValue(Src, VT, DAG);
     SDValue Store = DAG.getStore(Chain, Value,
-                                   getMemBasePlusOffset(Dst, DstOff, DAG),
-                                   DstSV, DstSVOff + DstOff);
+                                 getMemBasePlusOffset(Dst, DstOff, DAG),
+                                 DstSV, DstSVOff + DstOff);
     OutChains.push_back(Store);
     DstOff += VTSize;
   }
@@ -2962,9 +3139,10 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, SDValue Dst,
       return Chain;
 
     SDValue Result =
-      getMemcpyLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(),
+      getMemcpyLoadsAndStores(*this, Chain, Dst, Src,
+                              ConstantSize->getZExtValue(),
                               Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff);
-    if (Result.Val)
+    if (Result.getNode())
       return Result;
   }
 
@@ -2974,7 +3152,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, SDValue Dst,
     TLI.EmitTargetCodeForMemcpy(*this, Chain, Dst, Src, Size, Align,
                                 AlwaysInline,
                                 DstSV, DstSVOff, SrcSV, SrcSVOff);
-  if (Result.Val)
+  if (Result.getNode())
     return Result;
 
   // If we really need inline code and the target declined to provide it,
@@ -2982,7 +3160,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, SDValue Dst,
   if (AlwaysInline) {
     assert(ConstantSize && "AlwaysInline requires a constant size!");
     return getMemcpyLoadsAndStores(*this, Chain, Dst, Src,
-                                   ConstantSize->getValue(), Align, true,
+                                   ConstantSize->getZExtValue(), Align, true,
                                    DstSV, DstSVOff, SrcSV, SrcSVOff);
   }
 
@@ -2995,7 +3173,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, SDValue Dst,
   Entry.Node = Size; Args.push_back(Entry);
   std::pair<SDValue,SDValue> CallResult =
     TLI.LowerCallTo(Chain, Type::VoidTy,
-                    false, false, false, CallingConv::C, false,
+                    false, false, false, false, CallingConv::C, false,
                     getExternalSymbol("memcpy", TLI.getPointerTy()),
                     Args, *this);
   return CallResult.second;
@@ -3016,9 +3194,10 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, SDValue Dst,
       return Chain;
 
     SDValue Result =
-      getMemmoveLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(),
+      getMemmoveLoadsAndStores(*this, Chain, Dst, Src,
+                               ConstantSize->getZExtValue(),
                                Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff);
-    if (Result.Val)
+    if (Result.getNode())
       return Result;
   }
 
@@ -3027,7 +3206,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, SDValue Dst,
   SDValue Result =
     TLI.EmitTargetCodeForMemmove(*this, Chain, Dst, Src, Size, Align,
                                  DstSV, DstSVOff, SrcSV, SrcSVOff);
-  if (Result.Val)
+  if (Result.getNode())
     return Result;
 
   // Emit a library call.
@@ -3039,7 +3218,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, SDValue Dst,
   Entry.Node = Size; Args.push_back(Entry);
   std::pair<SDValue,SDValue> CallResult =
     TLI.LowerCallTo(Chain, Type::VoidTy,
-                    false, false, false, CallingConv::C, false,
+                    false, false, false, false, CallingConv::C, false,
                     getExternalSymbol("memmove", TLI.getPointerTy()),
                     Args, *this);
   return CallResult.second;
@@ -3059,9 +3238,9 @@ SDValue SelectionDAG::getMemset(SDValue Chain, SDValue Dst,
       return Chain;
 
     SDValue Result =
-      getMemsetStores(*this, Chain, Dst, Src, ConstantSize->getValue(), Align,
-                      DstSV, DstSVOff);
-    if (Result.Val)
+      getMemsetStores(*this, Chain, Dst, Src, ConstantSize->getZExtValue(),
+                      Align, DstSV, DstSVOff);
+    if (Result.getNode())
       return Result;
   }
 
@@ -3070,7 +3249,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, SDValue Dst,
   SDValue Result =
     TLI.EmitTargetCodeForMemset(*this, Chain, Dst, Src, Size, Align,
                                 DstSV, DstSVOff);
-  if (Result.Val)
+  if (Result.getNode())
     return Result;
 
   // Emit a library call.
@@ -3090,7 +3269,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, SDValue Dst,
   Args.push_back(Entry);
   std::pair<SDValue,SDValue> CallResult =
     TLI.LowerCallTo(Chain, Type::VoidTy,
-                    false, false, false, CallingConv::C, false,
+                    false, false, false, false, CallingConv::C, false,
                     getExternalSymbol("memset", TLI.getPointerTy()),
                     Args, *this);
   return CallResult.second;
@@ -3100,7 +3279,10 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDValue Chain,
                                 SDValue Ptr, SDValue Cmp, 
                                 SDValue Swp, const Value* PtrVal,
                                 unsigned Alignment) {
-  assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
+  assert((Opcode == ISD::ATOMIC_CMP_SWAP_8  ||
+          Opcode == ISD::ATOMIC_CMP_SWAP_16 ||
+          Opcode == ISD::ATOMIC_CMP_SWAP_32 ||
+          Opcode == ISD::ATOMIC_CMP_SWAP_64) && "Invalid Atomic Op");
   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
 
   MVT VT = Cmp.getValueType();
@@ -3126,13 +3308,50 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDValue Chain,
                                 SDValue Ptr, SDValue Val, 
                                 const Value* PtrVal,
                                 unsigned Alignment) {
-  assert((   Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB
-          || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_LOAD_AND
-          || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR
-          || Opcode == ISD::ATOMIC_LOAD_NAND 
-          || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
-          || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX) 
-         && "Invalid Atomic Op");
+  assert((Opcode == ISD::ATOMIC_LOAD_ADD_8 ||
+          Opcode == ISD::ATOMIC_LOAD_SUB_8 ||
+          Opcode == ISD::ATOMIC_LOAD_AND_8 ||
+          Opcode == ISD::ATOMIC_LOAD_OR_8 ||
+          Opcode == ISD::ATOMIC_LOAD_XOR_8 ||
+          Opcode == ISD::ATOMIC_LOAD_NAND_8 ||
+          Opcode == ISD::ATOMIC_LOAD_MIN_8 || 
+          Opcode == ISD::ATOMIC_LOAD_MAX_8 ||
+          Opcode == ISD::ATOMIC_LOAD_UMIN_8 || 
+          Opcode == ISD::ATOMIC_LOAD_UMAX_8 ||
+          Opcode == ISD::ATOMIC_SWAP_8 || 
+          Opcode == ISD::ATOMIC_LOAD_ADD_16 ||
+          Opcode == ISD::ATOMIC_LOAD_SUB_16 ||
+          Opcode == ISD::ATOMIC_LOAD_AND_16 ||
+          Opcode == ISD::ATOMIC_LOAD_OR_16 ||
+          Opcode == ISD::ATOMIC_LOAD_XOR_16 ||
+          Opcode == ISD::ATOMIC_LOAD_NAND_16 ||
+          Opcode == ISD::ATOMIC_LOAD_MIN_16 || 
+          Opcode == ISD::ATOMIC_LOAD_MAX_16 ||
+          Opcode == ISD::ATOMIC_LOAD_UMIN_16 || 
+          Opcode == ISD::ATOMIC_LOAD_UMAX_16 ||
+          Opcode == ISD::ATOMIC_SWAP_16 || 
+          Opcode == ISD::ATOMIC_LOAD_ADD_32 ||
+          Opcode == ISD::ATOMIC_LOAD_SUB_32 ||
+          Opcode == ISD::ATOMIC_LOAD_AND_32 ||
+          Opcode == ISD::ATOMIC_LOAD_OR_32 ||
+          Opcode == ISD::ATOMIC_LOAD_XOR_32 ||
+          Opcode == ISD::ATOMIC_LOAD_NAND_32 ||
+          Opcode == ISD::ATOMIC_LOAD_MIN_32 || 
+          Opcode == ISD::ATOMIC_LOAD_MAX_32 ||
+          Opcode == ISD::ATOMIC_LOAD_UMIN_32 || 
+          Opcode == ISD::ATOMIC_LOAD_UMAX_32 ||
+          Opcode == ISD::ATOMIC_SWAP_32 || 
+          Opcode == ISD::ATOMIC_LOAD_ADD_64 ||
+          Opcode == ISD::ATOMIC_LOAD_SUB_64 ||
+          Opcode == ISD::ATOMIC_LOAD_AND_64 ||
+          Opcode == ISD::ATOMIC_LOAD_OR_64 ||
+          Opcode == ISD::ATOMIC_LOAD_XOR_64 ||
+          Opcode == ISD::ATOMIC_LOAD_NAND_64 ||
+          Opcode == ISD::ATOMIC_LOAD_MIN_64 || 
+          Opcode == ISD::ATOMIC_LOAD_MAX_64 ||
+          Opcode == ISD::ATOMIC_LOAD_UMIN_64 || 
+          Opcode == ISD::ATOMIC_LOAD_UMAX_64 ||
+          Opcode == ISD::ATOMIC_SWAP_64)        && "Invalid Atomic Op");
 
   MVT VT = Val.getValueType();
 
@@ -3155,9 +3374,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDValue Chain,
 
 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
 /// Allowed to return something different (and simpler) if Simplify is true.
-SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps,
-                                     bool Simplify) {
-  if (Simplify && NumOps == 1)
+SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps) {
+  if (NumOps == 1)
     return Ops[0];
 
   SmallVector<MVT, 4> VTs;
@@ -3167,6 +3385,71 @@ SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps,
   return getNode(ISD::MERGE_VALUES, getVTList(&VTs[0], NumOps), Ops, NumOps);
 }
 
+SDValue
+SelectionDAG::getMemIntrinsicNode(unsigned Opcode,
+                                  const MVT *VTs, unsigned NumVTs,
+                                  const SDValue *Ops, unsigned NumOps,
+                                  MVT MemVT, const Value *srcValue, int SVOff,
+                                  unsigned Align, bool Vol,
+                                  bool ReadMem, bool WriteMem) {
+  return getMemIntrinsicNode(Opcode, makeVTList(VTs, NumVTs), Ops, NumOps,
+                             MemVT, srcValue, SVOff, Align, Vol,
+                             ReadMem, WriteMem);
+}
+
+SDValue
+SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDVTList VTList,
+                                  const SDValue *Ops, unsigned NumOps,
+                                  MVT MemVT, const Value *srcValue, int SVOff,
+                                  unsigned Align, bool Vol,
+                                  bool ReadMem, bool WriteMem) {
+  // Memoize the node unless it returns a flag.
+  MemIntrinsicSDNode *N;
+  if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) {
+    FoldingSetNodeID ID;
+    AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
+    void *IP = 0;
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+      return SDValue(E, 0);
+    
+    N = NodeAllocator.Allocate<MemIntrinsicSDNode>();
+    new (N) MemIntrinsicSDNode(Opcode, VTList, Ops, NumOps, MemVT,
+                               srcValue, SVOff, Align, Vol, ReadMem, WriteMem);
+    CSEMap.InsertNode(N, IP);
+  } else {
+    N = NodeAllocator.Allocate<MemIntrinsicSDNode>();
+    new (N) MemIntrinsicSDNode(Opcode, VTList, Ops, NumOps, MemVT,
+                               srcValue, SVOff, Align, Vol, ReadMem, WriteMem);
+  }
+  AllNodes.push_back(N);
+  return SDValue(N, 0);
+}
+
+SDValue
+SelectionDAG::getCall(unsigned CallingConv, bool IsVarArgs, bool IsTailCall,
+                      bool IsInreg, SDVTList VTs,
+                      const SDValue *Operands, unsigned NumOperands) {
+  // Do not include isTailCall in the folding set profile.
+  FoldingSetNodeID ID;
+  AddNodeIDNode(ID, ISD::CALL, VTs, Operands, NumOperands);
+  ID.AddInteger(CallingConv);
+  ID.AddInteger(IsVarArgs);
+  void *IP = 0;
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+    // Instead of including isTailCall in the folding set, we just
+    // set the flag of the existing node.
+    if (!IsTailCall)
+      cast<CallSDNode>(E)->setNotTailCall();
+    return SDValue(E, 0);
+  }
+  SDNode *N = NodeAllocator.Allocate<CallSDNode>();
+  new (N) CallSDNode(CallingConv, IsVarArgs, IsTailCall, IsInreg,
+                     VTs, Operands, NumOperands);
+  CSEMap.InsertNode(N, IP);
+  AllNodes.push_back(N);
+  return SDValue(N, 0);
+}
+
 SDValue
 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                       MVT VT, SDValue Chain,
@@ -3183,7 +3466,8 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
   } else {
     // Extending load.
     if (VT.isVector())
-      assert(EVT == VT.getVectorElementType() && "Invalid vector extload!");
+      assert(EVT.getVectorNumElements() == VT.getVectorNumElements() &&
+             "Invalid vector extload!");
     else
       assert(EVT.bitsLT(VT) &&
              "Should only be an extending load, not truncating!");
@@ -3205,8 +3489,7 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
   ID.AddInteger(AM);
   ID.AddInteger(ExtType);
   ID.AddInteger(EVT.getRawBits());
-  ID.AddInteger(Alignment);
-  ID.AddInteger(isVolatile);
+  ID.AddInteger(encodeMemSDNodeFlags(isVolatile, Alignment));
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
@@ -3265,8 +3548,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, SDValue Val,
   ID.AddInteger(ISD::UNINDEXED);
   ID.AddInteger(false);
   ID.AddInteger(VT.getRawBits());
-  ID.AddInteger(Alignment);
-  ID.AddInteger(isVolatile);
+  ID.AddInteger(encodeMemSDNodeFlags(isVolatile, Alignment));
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
@@ -3302,8 +3584,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, SDValue Val,
   ID.AddInteger(ISD::UNINDEXED);
   ID.AddInteger(1);
   ID.AddInteger(SVT.getRawBits());
-  ID.AddInteger(Alignment);
-  ID.AddInteger(isVolatile);
+  ID.AddInteger(encodeMemSDNodeFlags(isVolatile, Alignment));
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
@@ -3328,8 +3609,7 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, SDValue Base,
   ID.AddInteger(AM);
   ID.AddInteger(ST->isTruncatingStore());
   ID.AddInteger(ST->getMemoryVT().getRawBits());
-  ID.AddInteger(ST->getAlignment());
-  ID.AddInteger(ST->isVolatile());
+  ID.AddInteger(ST->getRawFlags());
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
@@ -3576,6 +3856,23 @@ SDVTList SelectionDAG::getVTList(MVT VT1, MVT VT2, MVT VT3) {
   return Result;
 }
 
+SDVTList SelectionDAG::getVTList(MVT VT1, MVT VT2, MVT VT3, MVT VT4) {
+  for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(),
+       E = VTList.rend(); I != E; ++I)
+    if (I->NumVTs == 4 && I->VTs[0] == VT1 && I->VTs[1] == VT2 &&
+                          I->VTs[2] == VT3 && I->VTs[3] == VT4)
+      return *I;
+
+  MVT *Array = Allocator.Allocate<MVT>(3);
+  Array[0] = VT1;
+  Array[1] = VT2;
+  Array[2] = VT3;
+  Array[3] = VT4;
+  SDVTList Result = makeVTList(Array, 4);
+  VTList.push_back(Result);
+  return Result;
+}
+
 SDVTList SelectionDAG::getVTList(const MVT *VTs, unsigned NumVTs) {
   switch (NumVTs) {
     case 0: assert(0 && "Cannot have nodes without results!");
@@ -3615,7 +3912,7 @@ SDVTList SelectionDAG::getVTList(const MVT *VTs, unsigned NumVTs) {
 /// input node is returned.  As a degenerate case, if you specify the same
 /// input operands as the node already has, the input node is returned.
 SDValue SelectionDAG::UpdateNodeOperands(SDValue InN, SDValue Op) {
-  SDNode *N = InN.Val;
+  SDNode *N = InN.getNode();
   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
   
   // Check to see if there is no change.
@@ -3624,17 +3921,18 @@ SDValue SelectionDAG::UpdateNodeOperands(SDValue InN, SDValue Op) {
   // See if the modified node already exists.
   void *InsertPos = 0;
   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
-    return SDValue(Existing, InN.ResNo);
+    return SDValue(Existing, InN.getResNo());
   
   // Nope it doesn't.  Remove the node from its current place in the maps.
   if (InsertPos)
-    RemoveNodeFromCSEMaps(N);
+    if (!RemoveNodeFromCSEMaps(N))
+      InsertPos = 0;
   
   // Now we update the operands.
   N->OperandList[0].getVal()->removeUser(0, N);
   N->OperandList[0] = Op;
   N->OperandList[0].setUser(N);
-  Op.Val->addUser(0, N);
+  Op.getNode()->addUser(0, N);
   
   // If this gets put into a CSE map, add it.
   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
@@ -3643,7 +3941,7 @@ SDValue SelectionDAG::UpdateNodeOperands(SDValue InN, SDValue Op) {
 
 SDValue SelectionDAG::
 UpdateNodeOperands(SDValue InN, SDValue Op1, SDValue Op2) {
-  SDNode *N = InN.Val;
+  SDNode *N = InN.getNode();
   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
   
   // Check to see if there is no change.
@@ -3653,24 +3951,25 @@ UpdateNodeOperands(SDValue InN, SDValue Op1, SDValue Op2) {
   // See if the modified node already exists.
   void *InsertPos = 0;
   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
-    return SDValue(Existing, InN.ResNo);
+    return SDValue(Existing, InN.getResNo());
   
   // Nope it doesn't.  Remove the node from its current place in the maps.
   if (InsertPos)
-    RemoveNodeFromCSEMaps(N);
+    if (!RemoveNodeFromCSEMaps(N))
+      InsertPos = 0;
   
   // Now we update the operands.
   if (N->OperandList[0] != Op1) {
     N->OperandList[0].getVal()->removeUser(0, N);
     N->OperandList[0] = Op1;
     N->OperandList[0].setUser(N);
-    Op1.Val->addUser(0, N);
+    Op1.getNode()->addUser(0, N);
   }
   if (N->OperandList[1] != Op2) {
     N->OperandList[1].getVal()->removeUser(1, N);
     N->OperandList[1] = Op2;
     N->OperandList[1].setUser(N);
-    Op2.Val->addUser(1, N);
+    Op2.getNode()->addUser(1, N);
   }
   
   // If this gets put into a CSE map, add it.
@@ -3700,7 +3999,7 @@ UpdateNodeOperands(SDValue N, SDValue Op1, SDValue Op2,
 
 SDValue SelectionDAG::
 UpdateNodeOperands(SDValue InN, const SDValue *Ops, unsigned NumOps) {
-  SDNode *N = InN.Val;
+  SDNode *N = InN.getNode();
   assert(N->getNumOperands() == NumOps &&
          "Update with wrong number of operands");
   
@@ -3719,11 +4018,12 @@ UpdateNodeOperands(SDValue InN, const SDValue *Ops, unsigned NumOps) {
   // See if the modified node already exists.
   void *InsertPos = 0;
   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos))
-    return SDValue(Existing, InN.ResNo);
+    return SDValue(Existing, InN.getResNo());
   
   // Nope it doesn't.  Remove the node from its current place in the maps.
   if (InsertPos)
-    RemoveNodeFromCSEMaps(N);
+    if (!RemoveNodeFromCSEMaps(N))
+      InsertPos = 0;
   
   // Now we update the operands.
   for (unsigned i = 0; i != NumOps; ++i) {
@@ -3731,7 +4031,7 @@ UpdateNodeOperands(SDValue InN, const SDValue *Ops, unsigned NumOps) {
       N->OperandList[i].getVal()->removeUser(i, N);
       N->OperandList[i] = Ops[i];
       N->OperandList[i].setUser(N);
-      Ops[i].Val->addUser(i, N);
+      Ops[i].getNode()->addUser(i, N);
     }
   }
 
@@ -3810,6 +4110,13 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
   return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
 }
 
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
+                                   MVT VT1, MVT VT2, MVT VT3, MVT VT4,
+                                   const SDValue *Ops, unsigned NumOps) {
+  SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
+  return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
+}
+
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 
                                    MVT VT1, MVT VT2,
                                    SDValue Op1) {
@@ -3835,6 +4142,15 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
   return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
 }
 
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
+                                   MVT VT1, MVT VT2, MVT VT3,
+                                   SDValue Op1, SDValue Op2, 
+                                   SDValue Op3) {
+  SDVTList VTs = getVTList(VT1, VT2, VT3);
+  SDValue Ops[] = { Op1, Op2, Op3 };
+  return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
+}
+
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                    SDVTList VTs, const SDValue *Ops,
                                    unsigned NumOps) {
@@ -3931,7 +4247,7 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
 ///
 /// Using MorphNodeTo is faster than creating a new node and swapping it in
 /// with ReplaceAllUsesWith both because it often avoids allocating a new
-/// node, and because it doesn't require CSE recalulation for any of
+/// node, and because it doesn't require CSE recalculation for any of
 /// the node's users.
 ///
 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
@@ -3946,7 +4262,8 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
       return ON;
   }
 
-  RemoveNodeFromCSEMaps(N);
+  if (!RemoveNodeFromCSEMaps(N))
+    IP = 0;
 
   // Start the morphing.
   N->NodeType = Opc;
@@ -3969,11 +4286,12 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
   if (NumOps > N->NumOperands) {
     if (N->OperandsNeedDelete)
       delete[] N->OperandList;
+
     if (N->isMachineOpcode()) {
       // We're creating a final node that will live unmorphed for the
-      // remainder of this SelectionDAG's duration, so we can allocate the
-      // operands directly out of the pool with no recycling metadata.
-      N->OperandList = Allocator.Allocate<SDUse>(NumOps);
+      // remainder of the current SelectionDAG iteration, so we can allocate
+      // the operands directly out of a pool with no recycling metadata.
+      N->OperandList = OperandAllocator.Allocate<SDUse>(NumOps);
       N->OperandsNeedDelete = false;
     } else {
       N->OperandList = new SDUse[NumOps];
@@ -3988,7 +4306,6 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
     N->OperandList[i].setUser(N);
     SDNode *ToUse = N->OperandList[i].getVal();
     ToUse->addUser(i, N);
-    DeadNodeSet.erase(ToUse);
   }
 
   // Delete any nodes that are still dead after adding the uses for the
@@ -4013,70 +4330,70 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
 /// node of the specified opcode and operands, it returns that node instead of
 /// the current one.
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT) {
-  return getNode(~Opcode, VT).Val;
+  return getNode(~Opcode, VT).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT, SDValue Op1) {
-  return getNode(~Opcode, VT, Op1).Val;
+  return getNode(~Opcode, VT, Op1).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
                                     SDValue Op1, SDValue Op2) {
-  return getNode(~Opcode, VT, Op1, Op2).Val;
+  return getNode(~Opcode, VT, Op1, Op2).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
                                     SDValue Op1, SDValue Op2,
                                     SDValue Op3) {
-  return getNode(~Opcode, VT, Op1, Op2, Op3).Val;
+  return getNode(~Opcode, VT, Op1, Op2, Op3).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
                                     const SDValue *Ops, unsigned NumOps) {
-  return getNode(~Opcode, VT, Ops, NumOps).Val;
+  return getNode(~Opcode, VT, Ops, NumOps).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2) {
   const MVT *VTs = getNodeValueTypes(VT1, VT2);
   SDValue Op;
-  return getNode(~Opcode, VTs, 2, &Op, 0).Val;
+  return getNode(~Opcode, VTs, 2, &Op, 0).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
                                     MVT VT2, SDValue Op1) {
   const MVT *VTs = getNodeValueTypes(VT1, VT2);
-  return getNode(~Opcode, VTs, 2, &Op1, 1).Val;
+  return getNode(~Opcode, VTs, 2, &Op1, 1).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
                                     MVT VT2, SDValue Op1,
                                     SDValue Op2) {
   const MVT *VTs = getNodeValueTypes(VT1, VT2);
   SDValue Ops[] = { Op1, Op2 };
-  return getNode(~Opcode, VTs, 2, Ops, 2).Val;
+  return getNode(~Opcode, VTs, 2, Ops, 2).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
                                     MVT VT2, SDValue Op1,
                                     SDValue Op2, SDValue Op3) {
   const MVT *VTs = getNodeValueTypes(VT1, VT2);
   SDValue Ops[] = { Op1, Op2, Op3 };
-  return getNode(~Opcode, VTs, 2, Ops, 3).Val;
+  return getNode(~Opcode, VTs, 2, Ops, 3).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2,
                                     const SDValue *Ops, unsigned NumOps) {
   const MVT *VTs = getNodeValueTypes(VT1, VT2);
-  return getNode(~Opcode, VTs, 2, Ops, NumOps).Val;
+  return getNode(~Opcode, VTs, 2, Ops, NumOps).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
                                     SDValue Op1, SDValue Op2) {
   const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
   SDValue Ops[] = { Op1, Op2 };
-  return getNode(~Opcode, VTs, 3, Ops, 2).Val;
+  return getNode(~Opcode, VTs, 3, Ops, 2).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
                                     SDValue Op1, SDValue Op2,
                                     SDValue Op3) {
   const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
   SDValue Ops[] = { Op1, Op2, Op3 };
-  return getNode(~Opcode, VTs, 3, Ops, 3).Val;
+  return getNode(~Opcode, VTs, 3, Ops, 3).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
                                     const SDValue *Ops, unsigned NumOps) {
   const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
-  return getNode(~Opcode, VTs, 3, Ops, NumOps).Val;
+  return getNode(~Opcode, VTs, 3, Ops, NumOps).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
                                     MVT VT2, MVT VT3, MVT VT4,
@@ -4087,14 +4404,14 @@ SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
   VTList.push_back(VT3);
   VTList.push_back(VT4);
   const MVT *VTs = getNodeValueTypes(VTList);
-  return getNode(~Opcode, VTs, 4, Ops, NumOps).Val;
+  return getNode(~Opcode, VTs, 4, Ops, NumOps).getNode();
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode,
                                     const std::vector<MVT> &ResultTys,
                                     const SDValue *Ops, unsigned NumOps) {
   const MVT *VTs = getNodeValueTypes(ResultTys);
   return getNode(~Opcode, VTs, ResultTys.size(),
-                 Ops, NumOps).Val;
+                 Ops, NumOps).getNode();
 }
 
 /// getNodeIfExists - Get the specified node if it's already available, or
@@ -4119,10 +4436,10 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
 ///
 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To,
                                       DAGUpdateListener *UpdateListener) {
-  SDNode *From = FromN.Val;
-  assert(From->getNumValues() == 1 && FromN.ResNo == 0 && 
+  SDNode *From = FromN.getNode();
+  assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 
          "Cannot replace with this method!");
-  assert(From != To.Val && "Cannot replace uses of with self");
+  assert(From != To.getNode() && "Cannot replace uses of with self");
 
   while (!From->use_empty()) {
     SDNode::use_iterator UI = From->use_begin();
@@ -4137,7 +4454,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To,
         From->removeUser(operandNum, U);
         *I = To;
         I->setUser(U);
-        To.Val->addUser(operandNum, U);
+        To.getNode()->addUser(operandNum, U);
       }    
 
     // Now that we have modified U, add it back to the CSE maps.  If it already
@@ -4184,7 +4501,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To,
          I != E; ++I, ++operandNum)
       if (I->getVal() == From) {
         From->removeUser(operandNum, U);
-        I->getVal() = To;
+        I->getSDValue().setNode(To);
         To->addUser(operandNum, U);
       }
 
@@ -4226,11 +4543,11 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From,
     for (SDNode::op_iterator I = U->op_begin(), E = U->op_end();
          I != E; ++I, ++operandNum)
       if (I->getVal() == From) {
-        const SDValue &ToOp = To[I->getSDValue().ResNo];
+        const SDValue &ToOp = To[I->getSDValue().getResNo()];
         From->removeUser(operandNum, U);
         *I = ToOp;
         I->setUser(U);
-        ToOp.Val->addUser(operandNum, U);
+        ToOp.getNode()->addUser(operandNum, U);
       }
 
     // Now that we have modified U, add it back to the CSE maps.  If it already
@@ -4251,7 +4568,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From,
 }
 
 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
-/// uses of other values produced by From.Val alone.  The Deleted vector is
+/// uses of other values produced by From.getVal() alone.  The Deleted vector is
 /// handled the same way as for ReplaceAllUsesWith.
 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To,
                                              DAGUpdateListener *UpdateListener){
@@ -4259,14 +4576,14 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To,
   if (From == To) return;
 
   // Handle the simple, trivial, case efficiently.
-  if (From.Val->getNumValues() == 1) {
+  if (From.getNode()->getNumValues() == 1) {
     ReplaceAllUsesWith(From, To, UpdateListener);
     return;
   }
 
-  // Get all of the users of From.Val.  We want these in a nice,
+  // Get all of the users of From.getNode().  We want these in a nice,
   // deterministically ordered and uniqued set, so we use a SmallSetVector.
-  SmallSetVector<SDNode*, 16> Users(From.Val->use_begin(), From.Val->use_end());
+  SmallSetVector<SDNode*, 16> Users(From.getNode()->use_begin(), From.getNode()->use_end());
 
   while (!Users.empty()) {
     // We know that this user uses some value of From.  If it is the right
@@ -4289,10 +4606,10 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To,
     // Update all operands that match "From" in case there are multiple uses.
     for (; Op != E; ++Op) {
       if (*Op == From) {
-        From.Val->removeUser(Op-User->op_begin(), User);
+        From.getNode()->removeUser(Op-User->op_begin(), User);
         *Op = To;
         Op->setUser(User);
-        To.Val->addUser(Op-User->op_begin(), User);
+        To.getNode()->addUser(Op-User->op_begin(), User);
       }
     }
                
@@ -4316,7 +4633,7 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To,
 }
 
 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
-/// uses of other values produced by From.Val alone.  The same value may
+/// uses of other values produced by From.getVal() alone.  The same value may
 /// appear in both the From and To list.  The Deleted vector is
 /// handled the same way as for ReplaceAllUsesWith.
 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
@@ -4329,8 +4646,8 @@ void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
 
   SmallVector<std::pair<SDNode *, unsigned>, 16> Users;
   for (unsigned i = 0; i != Num; ++i)
-    for (SDNode::use_iterator UI = From[i].Val->use_begin(), 
-         E = From[i].Val->use_end(); UI != E; ++UI)
+    for (SDNode::use_iterator UI = From[i].getNode()->use_begin(), 
+         E = From[i].getNode()->use_end(); UI != E; ++UI)
       Users.push_back(std::make_pair(*UI, i));
 
   while (!Users.empty()) {
@@ -4355,10 +4672,10 @@ void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
     // Update all operands that match "From" in case there are multiple uses.
     for (; Op != E; ++Op) {
       if (*Op == From[i]) {
-        From[i].Val->removeUser(Op-User->op_begin(), User);
+        From[i].getNode()->removeUser(Op-User->op_begin(), User);
         *Op = To[i];
         Op->setUser(User);
-        To[i].Val->addUser(Op-User->op_begin(), User);
+        To[i].getNode()->addUser(Op-User->op_begin(), User);
       }
     }
                
@@ -4384,43 +4701,74 @@ void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
 /// based on their topological order. It returns the maximum id and a vector
 /// of the SDNodes* in assigned order by reference.
-unsigned SelectionDAG::AssignTopologicalOrder(std::vector<SDNode*> &TopOrder) {
-  unsigned DAGSize = AllNodes.size();
-  std::vector<unsigned> InDegree(DAGSize);
-  std::vector<SDNode*> Sources;
-
-  // Use a two pass approach to avoid using a std::map which is slow.
-  unsigned Id = 0;
-  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I){
-    SDNode *N = I;
-    N->setNodeId(Id++);
-    unsigned Degree = N->use_size();
-    InDegree[N->getNodeId()] = Degree;
-    if (Degree == 0)
-      Sources.push_back(N);
-  }
-
-  TopOrder.clear();
-  TopOrder.reserve(DAGSize);
-  while (!Sources.empty()) {
-    SDNode *N = Sources.back();
-    Sources.pop_back();
-    TopOrder.push_back(N);
-    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
-      SDNode *P = I->getVal();
-      unsigned Degree = --InDegree[P->getNodeId()];
-      if (Degree == 0)
-        Sources.push_back(P);
+unsigned SelectionDAG::AssignTopologicalOrder() {
+
+  unsigned DAGSize = 0;
+
+  // SortedPos tracks the progress of the algorithm. Nodes before it are
+  // sorted, nodes after it are unsorted. When the algorithm completes
+  // it is at the end of the list.
+  allnodes_iterator SortedPos = allnodes_begin();
+
+  // Visit all the nodes. Move nodes with no operands to the front of
+  // the list immediately. Annotate nodes that do have operands with their
+  // operand count. Before we do this, the Node Id fields of the nodes
+  // may contain arbitrary values. After, the Node Id fields for nodes
+  // before SortedPos will contain the topological sort index, and the
+  // Node Id fields for nodes At SortedPos and after will contain the
+  // count of outstanding operands.
+  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
+    SDNode *N = I++;
+    unsigned Degree = N->getNumOperands();
+    if (Degree == 0) {
+      // A node with no uses, add it to the result array immediately.
+      N->setNodeId(DAGSize++);
+      allnodes_iterator Q = N;
+      if (Q != SortedPos)
+        SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
+      ++SortedPos;
+    } else {
+      // Temporarily use the Node Id as scratch space for the degree count.
+      N->setNodeId(Degree);
     }
   }
 
-  // Second pass, assign the actual topological order as node ids.
-  Id = 0;
-  for (std::vector<SDNode*>::iterator TI = TopOrder.begin(),TE = TopOrder.end();
-       TI != TE; ++TI)
-    (*TI)->setNodeId(Id++);
+  // Visit all the nodes. As we iterate, moves nodes into sorted order,
+  // such that by the time the end is reached all nodes will be sorted.
+  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
+    SDNode *N = I;
+    for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
+         UI != UE; ++UI) {
+      SDNode *P = *UI;
+      unsigned Degree = P->getNodeId();
+      --Degree;
+      if (Degree == 0) {
+        // All of P's operands are sorted, so P may sorted now.
+        P->setNodeId(DAGSize++);
+        if (P != SortedPos)
+          SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
+        ++SortedPos;
+      } else {
+        // Update P's outstanding operand count.
+        P->setNodeId(Degree);
+      }
+    }
+  }
 
-  return Id;
+  assert(SortedPos == AllNodes.end() &&
+         "Topological sort incomplete!");
+  assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
+         "First node in topological sort is not the entry token!");
+  assert(AllNodes.front().getNodeId() == 0 &&
+         "First node in topological sort has non-zero id!");
+  assert(AllNodes.front().getNumOperands() == 0 &&
+         "First node in topological sort has operands!");
+  assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
+         "Last node in topologic sort has unexpected id!");
+  assert(AllNodes.back().use_empty() &&
+         "Last node in topologic sort has users!");
+  assert(DAGSize == allnodes_size() && "Node count mismatch!");
+  return DAGSize;
 }
 
 
@@ -4455,13 +4803,16 @@ void MemSDNode::ANCHOR() {}
 void LoadSDNode::ANCHOR() {}
 void StoreSDNode::ANCHOR() {}
 void AtomicSDNode::ANCHOR() {}
+void MemIntrinsicSDNode::ANCHOR() {}
+void CallSDNode::ANCHOR() {}
+void CvtRndSatSDNode::ANCHOR() {}
 
 HandleSDNode::~HandleSDNode() {
   DropOperands();
 }
 
 GlobalAddressSDNode::GlobalAddressSDNode(bool isTarget, const GlobalValue *GA,
-                                         MVT VT, int o)
+                                         MVT VT, int64_t o)
   : SDNode(isa<GlobalVariable>(GA) &&
            cast<GlobalVariable>(GA)->isThreadLocal() ?
            // Thread Local
@@ -4476,32 +4827,48 @@ MemSDNode::MemSDNode(unsigned Opc, SDVTList VTs, MVT memvt,
                      const Value *srcValue, int SVO,
                      unsigned alignment, bool vol)
  : SDNode(Opc, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO),
-   Flags(vol | ((Log2_32(alignment) + 1) << 1)) {
+   Flags(encodeMemSDNodeFlags(vol, alignment)) {
 
   assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!");
   assert(getAlignment() == alignment && "Alignment representation error!");
   assert(isVolatile() == vol && "Volatile representation error!");
 }
 
+MemSDNode::MemSDNode(unsigned Opc, SDVTList VTs, const SDValue *Ops,
+                     unsigned NumOps, MVT memvt, const Value *srcValue,
+                     int SVO, unsigned alignment, bool vol)
+   : SDNode(Opc, VTs, Ops, NumOps),
+     MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO),
+     Flags(vol | ((Log2_32(alignment) + 1) << 1)) {
+  assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!");
+  assert(getAlignment() == alignment && "Alignment representation error!");
+  assert(isVolatile() == vol && "Volatile representation error!");
+}
+
 /// getMemOperand - Return a MachineMemOperand object describing the memory
 /// reference performed by this memory reference.
 MachineMemOperand MemSDNode::getMemOperand() const {
-  int Flags;
+  int Flags = 0;
   if (isa<LoadSDNode>(this))
     Flags = MachineMemOperand::MOLoad;
   else if (isa<StoreSDNode>(this))
     Flags = MachineMemOperand::MOStore;
-  else {
-    assert(isa<AtomicSDNode>(this) && "Unknown MemSDNode opcode!");
+  else if (isa<AtomicSDNode>(this)) {
     Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
   }
+  else {
+    const MemIntrinsicSDNode* MemIntrinNode = dyn_cast<MemIntrinsicSDNode>(this);
+    assert(MemIntrinNode && "Unknown MemSDNode opcode!");
+    if (MemIntrinNode->readMem()) Flags |= MachineMemOperand::MOLoad;
+    if (MemIntrinNode->writeMem()) Flags |= MachineMemOperand::MOStore;
+  }
 
   int Size = (getMemoryVT().getSizeInBits() + 7) >> 3;
   if (isVolatile()) Flags |= MachineMemOperand::MOVolatile;
   
   // Check if the memory reference references a frame index
   const FrameIndexSDNode *FI = 
-  dyn_cast<const FrameIndexSDNode>(getBasePtr().Val);
+  dyn_cast<const FrameIndexSDNode>(getBasePtr().getNode());
   if (!getSrcValue() && FI)
     return MachineMemOperand(PseudoSourceValue::getFixedStack(FI->getIndex()),
                              Flags, 0, Size, getAlignment());
@@ -4512,7 +4879,7 @@ MachineMemOperand MemSDNode::getMemOperand() const {
 
 /// Profile - Gather unique data for the node.
 ///
-void SDNode::Profile(FoldingSetNodeID &ID) {
+void SDNode::Profile(FoldingSetNodeID &ID) const {
   AddNodeIDNode(ID, this);
 }
 
@@ -4537,7 +4904,7 @@ bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
 
   // TODO: Only iterate over uses of a given value of the node
   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
-    if (UI.getUse().getSDValue().ResNo == Value) {
+    if (UI.getUse().getSDValue().getResNo() == Value) {
       if (NUses == 0)
         return false;
       --NUses;
@@ -4555,7 +4922,7 @@ bool SDNode::hasAnyUseOfValue(unsigned Value) const {
   assert(Value < getNumValues() && "Bad value!");
 
   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
-    if (UI.getUse().getSDValue().ResNo == Value)
+    if (UI.getUse().getSDValue().getResNo() == Value)
       return true;
 
   return false;
@@ -4630,7 +4997,7 @@ static void findPredecessor(SDNode *N, const SDNode *P, bool &found,
     return;
 
   for (unsigned i = 0, e = N->getNumOperands(); !found && i != e; ++i) {
-    SDNode *Op = N->getOperand(i).Val;
+    SDNode *Op = N->getOperand(i).getNode();
     if (Op == P) {
       found = true;
       return;
@@ -4652,7 +5019,7 @@ bool SDNode::isPredecessorOf(SDNode *N) const {
 
 uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
   assert(Num < NumOperands && "Invalid child # of SDNode!");
-  return cast<ConstantSDNode>(OperandList[Num])->getValue();
+  return cast<ConstantSDNode>(OperandList[Num])->getZExtValue();
 }
 
 std::string SDNode::getOperationName(const SelectionDAG *G) const {
@@ -4681,18 +5048,54 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
 #endif
   case ISD::PREFETCH:      return "Prefetch";
   case ISD::MEMBARRIER:    return "MemBarrier";
-  case ISD::ATOMIC_CMP_SWAP:  return "AtomicCmpSwap";
-  case ISD::ATOMIC_LOAD_ADD:  return "AtomicLoadAdd";
-  case ISD::ATOMIC_LOAD_SUB:  return "AtomicLoadSub";
-  case ISD::ATOMIC_LOAD_AND:  return "AtomicLoadAnd";
-  case ISD::ATOMIC_LOAD_OR:   return "AtomicLoadOr";
-  case ISD::ATOMIC_LOAD_XOR:  return "AtomicLoadXor";
-  case ISD::ATOMIC_LOAD_NAND: return "AtomicLoadNand";
-  case ISD::ATOMIC_LOAD_MIN:  return "AtomicLoadMin";
-  case ISD::ATOMIC_LOAD_MAX:  return "AtomicLoadMax";
-  case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin";
-  case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax";
-  case ISD::ATOMIC_SWAP:   return "AtomicSWAP";
+  case ISD::ATOMIC_CMP_SWAP_8:  return "AtomicCmpSwap8";
+  case ISD::ATOMIC_SWAP_8:      return "AtomicSwap8";
+  case ISD::ATOMIC_LOAD_ADD_8:  return "AtomicLoadAdd8";
+  case ISD::ATOMIC_LOAD_SUB_8:  return "AtomicLoadSub8";
+  case ISD::ATOMIC_LOAD_AND_8:  return "AtomicLoadAnd8";
+  case ISD::ATOMIC_LOAD_OR_8:   return "AtomicLoadOr8";
+  case ISD::ATOMIC_LOAD_XOR_8:  return "AtomicLoadXor8";
+  case ISD::ATOMIC_LOAD_NAND_8: return "AtomicLoadNand8";
+  case ISD::ATOMIC_LOAD_MIN_8:  return "AtomicLoadMin8";
+  case ISD::ATOMIC_LOAD_MAX_8:  return "AtomicLoadMax8";
+  case ISD::ATOMIC_LOAD_UMIN_8: return "AtomicLoadUMin8";
+  case ISD::ATOMIC_LOAD_UMAX_8: return "AtomicLoadUMax8";
+  case ISD::ATOMIC_CMP_SWAP_16:  return "AtomicCmpSwap16";
+  case ISD::ATOMIC_SWAP_16:      return "AtomicSwap16";
+  case ISD::ATOMIC_LOAD_ADD_16:  return "AtomicLoadAdd16";
+  case ISD::ATOMIC_LOAD_SUB_16:  return "AtomicLoadSub16";
+  case ISD::ATOMIC_LOAD_AND_16:  return "AtomicLoadAnd16";
+  case ISD::ATOMIC_LOAD_OR_16:   return "AtomicLoadOr16";
+  case ISD::ATOMIC_LOAD_XOR_16:  return "AtomicLoadXor16";
+  case ISD::ATOMIC_LOAD_NAND_16: return "AtomicLoadNand16";
+  case ISD::ATOMIC_LOAD_MIN_16:  return "AtomicLoadMin16";
+  case ISD::ATOMIC_LOAD_MAX_16:  return "AtomicLoadMax16";
+  case ISD::ATOMIC_LOAD_UMIN_16: return "AtomicLoadUMin16";
+  case ISD::ATOMIC_LOAD_UMAX_16: return "AtomicLoadUMax16";
+  case ISD::ATOMIC_CMP_SWAP_32:  return "AtomicCmpSwap32";
+  case ISD::ATOMIC_SWAP_32:      return "AtomicSwap32";
+  case ISD::ATOMIC_LOAD_ADD_32:  return "AtomicLoadAdd32";
+  case ISD::ATOMIC_LOAD_SUB_32:  return "AtomicLoadSub32";
+  case ISD::ATOMIC_LOAD_AND_32:  return "AtomicLoadAnd32";
+  case ISD::ATOMIC_LOAD_OR_32:   return "AtomicLoadOr32";
+  case ISD::ATOMIC_LOAD_XOR_32:  return "AtomicLoadXor32";
+  case ISD::ATOMIC_LOAD_NAND_32: return "AtomicLoadNand32";
+  case ISD::ATOMIC_LOAD_MIN_32:  return "AtomicLoadMin32";
+  case ISD::ATOMIC_LOAD_MAX_32:  return "AtomicLoadMax32";
+  case ISD::ATOMIC_LOAD_UMIN_32: return "AtomicLoadUMin32";
+  case ISD::ATOMIC_LOAD_UMAX_32: return "AtomicLoadUMax32";
+  case ISD::ATOMIC_CMP_SWAP_64:  return "AtomicCmpSwap64";
+  case ISD::ATOMIC_SWAP_64:      return "AtomicSwap64";
+  case ISD::ATOMIC_LOAD_ADD_64:  return "AtomicLoadAdd64";
+  case ISD::ATOMIC_LOAD_SUB_64:  return "AtomicLoadSub64";
+  case ISD::ATOMIC_LOAD_AND_64:  return "AtomicLoadAnd64";
+  case ISD::ATOMIC_LOAD_OR_64:   return "AtomicLoadOr64";
+  case ISD::ATOMIC_LOAD_XOR_64:  return "AtomicLoadXor64";
+  case ISD::ATOMIC_LOAD_NAND_64: return "AtomicLoadNand64";
+  case ISD::ATOMIC_LOAD_MIN_64:  return "AtomicLoadMin64";
+  case ISD::ATOMIC_LOAD_MAX_64:  return "AtomicLoadMax64";
+  case ISD::ATOMIC_LOAD_UMIN_64: return "AtomicLoadUMin64";
+  case ISD::ATOMIC_LOAD_UMAX_64: return "AtomicLoadUMax64";
   case ISD::PCMARKER:      return "PCMarker";
   case ISD::READCYCLECOUNTER: return "ReadCycleCounter";
   case ISD::SRCVALUE:      return "SrcValue";
@@ -4723,12 +5126,12 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::ConstantPool:  return "ConstantPool";
   case ISD::ExternalSymbol: return "ExternalSymbol";
   case ISD::INTRINSIC_WO_CHAIN: {
-    unsigned IID = cast<ConstantSDNode>(getOperand(0))->getValue();
+    unsigned IID = cast<ConstantSDNode>(getOperand(0))->getZExtValue();
     return Intrinsic::getName((Intrinsic::ID)IID);
   }
   case ISD::INTRINSIC_VOID:
   case ISD::INTRINSIC_W_CHAIN: {
-    unsigned IID = cast<ConstantSDNode>(getOperand(1))->getValue();
+    unsigned IID = cast<ConstantSDNode>(getOperand(1))->getZExtValue();
     return Intrinsic::getName((Intrinsic::ID)IID);
   }
 
@@ -4762,6 +5165,11 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::FCOS:   return "fcos";
   case ISD::FPOWI:  return "fpowi";
   case ISD::FPOW:   return "fpow";
+  case ISD::FTRUNC: return "ftrunc";
+  case ISD::FFLOOR: return "ffloor";
+  case ISD::FCEIL:  return "fceil";
+  case ISD::FRINT:  return "frint";
+  case ISD::FNEARBYINT: return "fnearbyint";
 
   // Binary operators
   case ISD::ADD:    return "add";
@@ -4776,7 +5184,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::SMUL_LOHI:  return "smul_lohi";
   case ISD::UMUL_LOHI:  return "umul_lohi";
   case ISD::SDIVREM:    return "sdivrem";
-  case ISD::UDIVREM:    return "divrem";
+  case ISD::UDIVREM:    return "udivrem";
   case ISD::AND:    return "and";
   case ISD::OR:     return "or";
   case ISD::XOR:    return "xor";
@@ -4806,6 +5214,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::CARRY_FALSE:         return "carry_false";
   case ISD::ADDC:        return "addc";
   case ISD::ADDE:        return "adde";
+  case ISD::SADDO:       return "saddo";
+  case ISD::UADDO:       return "uaddo";
   case ISD::SUBC:        return "subc";
   case ISD::SUBE:        return "sube";
   case ISD::SHL_PARTS:   return "shl_parts";
@@ -4831,6 +5241,21 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::FP_TO_SINT:  return "fp_to_sint";
   case ISD::FP_TO_UINT:  return "fp_to_uint";
   case ISD::BIT_CONVERT: return "bit_convert";
+  
+  case ISD::CONVERT_RNDSAT: {
+    switch (cast<CvtRndSatSDNode>(this)->getCvtCode()) {
+    default: assert(0 && "Unknown cvt code!");
+    case ISD::CVT_FF:  return "cvt_ff";
+    case ISD::CVT_FS:  return "cvt_fs";
+    case ISD::CVT_FU:  return "cvt_fu";
+    case ISD::CVT_SF:  return "cvt_sf";
+    case ISD::CVT_UF:  return "cvt_uf";
+    case ISD::CVT_SS:  return "cvt_ss";
+    case ISD::CVT_SU:  return "cvt_su";
+    case ISD::CVT_US:  return "cvt_us";
+    case ISD::CVT_UU:  return "cvt_uu";
+    }
+  }
 
     // Control flow instructions
   case ISD::BR:      return "br";
@@ -4939,178 +5364,177 @@ std::string ISD::ArgFlagsTy::getArgFlagsString() {
 
 void SDNode::dump() const { dump(0); }
 void SDNode::dump(const SelectionDAG *G) const {
-  cerr << (void*)this << ": ";
+  print(errs(), G);
+  errs().flush();
+}
+
+void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const {
+  OS << (void*)this << ": ";
 
   for (unsigned i = 0, e = getNumValues(); i != e; ++i) {
-    if (i) cerr << ",";
+    if (i) OS << ",";
     if (getValueType(i) == MVT::Other)
-      cerr << "ch";
+      OS << "ch";
     else
-      cerr << getValueType(i).getMVTString();
+      OS << getValueType(i).getMVTString();
   }
-  cerr << " = " << getOperationName(G);
+  OS << " = " << getOperationName(G);
 
-  cerr << " ";
+  OS << " ";
   for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
-    if (i) cerr << ", ";
-    cerr << (void*)getOperand(i).Val;
-    if (unsigned RN = getOperand(i).ResNo)
-      cerr << ":" << RN;
+    if (i) OS << ", ";
+    OS << (void*)getOperand(i).getNode();
+    if (unsigned RN = getOperand(i).getResNo())
+      OS << ":" << RN;
   }
 
   if (!isTargetOpcode() && getOpcode() == ISD::VECTOR_SHUFFLE) {
-    SDNode *Mask = getOperand(2).Val;
-    cerr << "<";
+    SDNode *Mask = getOperand(2).getNode();
+    OS << "<";
     for (unsigned i = 0, e = Mask->getNumOperands(); i != e; ++i) {
-      if (i) cerr << ",";
+      if (i) OS << ",";
       if (Mask->getOperand(i).getOpcode() == ISD::UNDEF)
-        cerr << "u";
+        OS << "u";
       else
-        cerr << cast<ConstantSDNode>(Mask->getOperand(i))->getValue();
+        OS << cast<ConstantSDNode>(Mask->getOperand(i))->getZExtValue();
     }
-    cerr << ">";
+    OS << ">";
   }
 
   if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(this)) {
-    cerr << "<" << CSDN->getAPIntValue().toStringUnsigned() << ">";
+    OS << '<' << CSDN->getAPIntValue() << '>';
   } else if (const ConstantFPSDNode *CSDN = dyn_cast<ConstantFPSDNode>(this)) {
     if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEsingle)
-      cerr << "<" << CSDN->getValueAPF().convertToFloat() << ">";
+      OS << '<' << CSDN->getValueAPF().convertToFloat() << '>';
     else if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEdouble)
-      cerr << "<" << CSDN->getValueAPF().convertToDouble() << ">";
+      OS << '<' << CSDN->getValueAPF().convertToDouble() << '>';
     else {
-      cerr << "<APFloat(";
-      CSDN->getValueAPF().convertToAPInt().dump();
-      cerr << ")>";
+      OS << "<APFloat(";
+      CSDN->getValueAPF().bitcastToAPInt().dump();
+      OS << ")>";
     }
   } else if (const GlobalAddressSDNode *GADN =
              dyn_cast<GlobalAddressSDNode>(this)) {
-    int offset = GADN->getOffset();
-    cerr << "<";
-    WriteAsOperand(*cerr.stream(), GADN->getGlobal()) << ">";
+    int64_t offset = GADN->getOffset();
+    OS << '<';
+    WriteAsOperand(OS, GADN->getGlobal());
+    OS << '>';
     if (offset > 0)
-      cerr << " + " << offset;
+      OS << " + " << offset;
     else
-      cerr << " " << offset;
+      OS << " " << offset;
   } else if (const FrameIndexSDNode *FIDN = dyn_cast<FrameIndexSDNode>(this)) {
-    cerr << "<" << FIDN->getIndex() << ">";
+    OS << "<" << FIDN->getIndex() << ">";
   } else if (const JumpTableSDNode *JTDN = dyn_cast<JumpTableSDNode>(this)) {
-    cerr << "<" << JTDN->getIndex() << ">";
+    OS << "<" << JTDN->getIndex() << ">";
   } else if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(this)){
     int offset = CP->getOffset();
     if (CP->isMachineConstantPoolEntry())
-      cerr << "<" << *CP->getMachineCPVal() << ">";
+      OS << "<" << *CP->getMachineCPVal() << ">";
     else
-      cerr << "<" << *CP->getConstVal() << ">";
+      OS << "<" << *CP->getConstVal() << ">";
     if (offset > 0)
-      cerr << " + " << offset;
+      OS << " + " << offset;
     else
-      cerr << " " << offset;
+      OS << " " << offset;
   } else if (const BasicBlockSDNode *BBDN = dyn_cast<BasicBlockSDNode>(this)) {
-    cerr << "<";
+    OS << "<";
     const Value *LBB = (const Value*)BBDN->getBasicBlock()->getBasicBlock();
     if (LBB)
-      cerr << LBB->getName() << " ";
-    cerr << (const void*)BBDN->getBasicBlock() << ">";
+      OS << LBB->getName() << " ";
+    OS << (const void*)BBDN->getBasicBlock() << ">";
   } else if (const RegisterSDNode *R = dyn_cast<RegisterSDNode>(this)) {
     if (G && R->getReg() &&
         TargetRegisterInfo::isPhysicalRegister(R->getReg())) {
-      cerr << " " << G->getTarget().getRegisterInfo()->getName(R->getReg());
+      OS << " " << G->getTarget().getRegisterInfo()->getName(R->getReg());
     } else {
-      cerr << " #" << R->getReg();
+      OS << " #" << R->getReg();
     }
   } else if (const ExternalSymbolSDNode *ES =
              dyn_cast<ExternalSymbolSDNode>(this)) {
-    cerr << "'" << ES->getSymbol() << "'";
+    OS << "'" << ES->getSymbol() << "'";
   } else if (const SrcValueSDNode *M = dyn_cast<SrcValueSDNode>(this)) {
     if (M->getValue())
-      cerr << "<" << M->getValue() << ">";
+      OS << "<" << M->getValue() << ">";
     else
-      cerr << "<null>";
+      OS << "<null>";
   } else if (const MemOperandSDNode *M = dyn_cast<MemOperandSDNode>(this)) {
     if (M->MO.getValue())
-      cerr << "<" << M->MO.getValue() << ":" << M->MO.getOffset() << ">";
+      OS << "<" << M->MO.getValue() << ":" << M->MO.getOffset() << ">";
     else
-      cerr << "<null:" << M->MO.getOffset() << ">";
+      OS << "<null:" << M->MO.getOffset() << ">";
   } else if (const ARG_FLAGSSDNode *N = dyn_cast<ARG_FLAGSSDNode>(this)) {
-    cerr << N->getArgFlags().getArgFlagsString();
+    OS << N->getArgFlags().getArgFlagsString();
   } else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) {
-    cerr << ":" << N->getVT().getMVTString();
+    OS << ":" << N->getVT().getMVTString();
   }
   else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) {
     const Value *SrcValue = LD->getSrcValue();
     int SrcOffset = LD->getSrcValueOffset();
-    cerr << " <";
+    OS << " <";
     if (SrcValue)
-      cerr << SrcValue;
+      OS << SrcValue;
     else
-      cerr << "null";
-    cerr << ":" << SrcOffset << ">";
+      OS << "null";
+    OS << ":" << SrcOffset << ">";
 
     bool doExt = true;
     switch (LD->getExtensionType()) {
     default: doExt = false; break;
-    case ISD::EXTLOAD:
-      cerr << " <anyext ";
-      break;
-    case ISD::SEXTLOAD:
-      cerr << " <sext ";
-      break;
-    case ISD::ZEXTLOAD:
-      cerr << " <zext ";
-      break;
+    case ISD::EXTLOAD: OS << " <anyext "; break;
+    case ISD::SEXTLOAD: OS << " <sext "; break;
+    case ISD::ZEXTLOAD: OS << " <zext "; break;
     }
     if (doExt)
-      cerr << LD->getMemoryVT().getMVTString() << ">";
+      OS << LD->getMemoryVT().getMVTString() << ">";
 
     const char *AM = getIndexedModeName(LD->getAddressingMode());
     if (*AM)
-      cerr << " " << AM;
+      OS << " " << AM;
     if (LD->isVolatile())
-      cerr << " <volatile>";
-    cerr << " alignment=" << LD->getAlignment();
+      OS << " <volatile>";
+    OS << " alignment=" << LD->getAlignment();
   } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(this)) {
     const Value *SrcValue = ST->getSrcValue();
     int SrcOffset = ST->getSrcValueOffset();
-    cerr << " <";
+    OS << " <";
     if (SrcValue)
-      cerr << SrcValue;
+      OS << SrcValue;
     else
-      cerr << "null";
-    cerr << ":" << SrcOffset << ">";
+      OS << "null";
+    OS << ":" << SrcOffset << ">";
 
     if (ST->isTruncatingStore())
-      cerr << " <trunc "
-           << ST->getMemoryVT().getMVTString() << ">";
+      OS << " <trunc " << ST->getMemoryVT().getMVTString() << ">";
 
     const char *AM = getIndexedModeName(ST->getAddressingMode());
     if (*AM)
-      cerr << " " << AM;
+      OS << " " << AM;
     if (ST->isVolatile())
-      cerr << " <volatile>";
-    cerr << " alignment=" << ST->getAlignment();
+      OS << " <volatile>";
+    OS << " alignment=" << ST->getAlignment();
   } else if (const AtomicSDNode* AT = dyn_cast<AtomicSDNode>(this)) {
     const Value *SrcValue = AT->getSrcValue();
     int SrcOffset = AT->getSrcValueOffset();
-    cerr << " <";
+    OS << " <";
     if (SrcValue)
-      cerr << SrcValue;
+      OS << SrcValue;
     else
-      cerr << "null";
-    cerr << ":" << SrcOffset << ">";
+      OS << "null";
+    OS << ":" << SrcOffset << ">";
     if (AT->isVolatile())
-      cerr << " <volatile>";
-    cerr << " alignment=" << AT->getAlignment();
+      OS << " <volatile>";
+    OS << " alignment=" << AT->getAlignment();
   }
 }
 
 static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) {
   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-    if (N->getOperand(i).Val->hasOneUse())
-      DumpNodes(N->getOperand(i).Val, indent+2, G);
+    if (N->getOperand(i).getNode()->hasOneUse())
+      DumpNodes(N->getOperand(i).getNode(), indent+2, G);
     else
       cerr << "\n" << std::string(indent+2, ' ')
-           << (void*)N->getOperand(i).Val << ": <multiple use>";
+           << (void*)N->getOperand(i).getNode() << ": <multiple use>";
 
 
   cerr << "\n" << std::string(indent, ' ');
@@ -5123,11 +5547,11 @@ void SelectionDAG::dump() const {
   for (allnodes_const_iterator I = allnodes_begin(), E = allnodes_end();
        I != E; ++I) {
     const SDNode *N = I;
-    if (!N->hasOneUse() && N != getRoot().Val)
+    if (!N->hasOneUse() && N != getRoot().getNode())
       DumpNodes(N, 2, this);
   }
 
-  if (getRoot().Val) DumpNodes(getRoot().Val, 2, this);
+  if (getRoot().getNode()) DumpNodes(getRoot().getNode(), 2, this);
 
   cerr << "\n\n";
 }