From: Nate Begeman Date: Tue, 15 Sep 2009 19:05:41 +0000 (+0000) Subject: Better solution for tracking both the original alignment of the access, and the curre... X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=101b25c028706c61c7dd9fb92d0b3c1541cb12b6;p=oota-llvm.git Better solution for tracking both the original alignment of the access, and the current alignment based on the source value offset. This avoids increasing the size of mem nodes. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@81897 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 064fc874551..d6dc7c274a1 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -544,7 +544,7 @@ public: /// SDValue getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, bool isVolatile=false, - unsigned Alignment=0, unsigned OrigAlignment=0); + unsigned Alignment=0); SDValue getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, EVT EVT, bool isVolatile=false, @@ -554,14 +554,13 @@ public: SDValue getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType, EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset, const Value *SV, int SVOffset, EVT EVT, - bool isVolatile=false, unsigned Alignment=0, - unsigned OrigAlignment=0); + bool isVolatile=false, unsigned Alignment=0); /// getStore - Helper function to build ISD::STORE nodes. /// SDValue getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, bool isVolatile=false, - unsigned Alignment=0, unsigned OrigAlignment=0); + unsigned Alignment=0); SDValue getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, EVT TVT, bool isVolatile=false, unsigned Alignment=0); diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index a3ce5d432d3..04e1b853fe7 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -28,6 +28,7 @@ #include "llvm/CodeGen/ValueTypes.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/Support/Allocator.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/RecyclingAllocator.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/DebugLoc.h" @@ -1518,23 +1519,22 @@ private: //! SVOffset - Memory location offset. Note that base is defined in MemSDNode int SVOffset; - - //! OrigAlign - The original alignment of this MemSDNode in the case where - // this node was created by legalize from a MemSDNode with known alignment. - unsigned OrigAlign; - public: MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT MemoryVT, - const Value *srcValue, int SVOff, - unsigned alignment, bool isvolatile, unsigned oalign); + const Value *srcValue, int SVOff, unsigned alignment, + bool isvolatile); MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, unsigned NumOps, EVT MemoryVT, const Value *srcValue, int SVOff, - unsigned alignment, bool isvolatile, unsigned oalign); + unsigned alignment, bool isvolatile); /// Returns alignment and volatility of the memory access - unsigned getAlignment() const { return (1u << (SubclassData >> 6)) >> 1; } - unsigned getOriginalAlignment() const { return OrigAlign; } + unsigned getOriginalAlignment() const { + return (1u << (SubclassData >> 6)) >> 1; + } + unsigned getAlignment() const { + return MinAlign(getOriginalAlignment(), SVOffset); + } bool isVolatile() const { return (SubclassData >> 5) & 1; } /// getRawSubclassData - Return the SubclassData value, which contains an @@ -1605,14 +1605,14 @@ public: SDValue Cmp, SDValue Swp, const Value* SrcVal, unsigned Align=0) : MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0, - Align, /*isVolatile=*/true, /* OrigAlign=*/0) { + Align, /*isVolatile=*/true) { InitOperands(Ops, Chain, Ptr, Cmp, Swp); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value* SrcVal, unsigned Align=0) : MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0, - Align, /*isVolatile=*/true, /* OrigAlign=*/0) { + Align, /*isVolatile=*/true) { InitOperands(Ops, Chain, Ptr, Val); } @@ -1653,8 +1653,7 @@ public: const SDValue *Ops, unsigned NumOps, EVT MemoryVT, const Value *srcValue, int SVO, unsigned Align, bool Vol, bool ReadMem, bool WriteMem) - : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol, - /* OrigAlign=*/0), + : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol), ReadMem(ReadMem), WriteMem(WriteMem) { } @@ -2269,9 +2268,8 @@ class LSBaseSDNode : public MemSDNode { public: LSBaseSDNode(ISD::NodeType NodeTy, DebugLoc dl, SDValue *Operands, unsigned numOperands, SDVTList VTs, ISD::MemIndexedMode AM, - EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol, - unsigned OAlign) - : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol, OAlign) { + EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol) + : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol) { assert(Align != 0 && "Loads and stores should have non-zero aligment"); SubclassData |= AM << 2; assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); @@ -2309,10 +2307,9 @@ class LoadSDNode : public LSBaseSDNode { friend class SelectionDAG; LoadSDNode(SDValue *ChainPtrOff, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT LVT, - const Value *SV, int O=0, unsigned Align=0, bool Vol=false, - unsigned OAlign=0) + const Value *SV, int O=0, unsigned Align=0, bool Vol=false) : LSBaseSDNode(ISD::LOAD, dl, ChainPtrOff, 3, - VTs, AM, LVT, SV, O, Align, Vol, OAlign) { + VTs, AM, LVT, SV, O, Align, Vol) { SubclassData |= (unsigned short)ETy; assert(getExtensionType() == ETy && "LoadExtType encoding error!"); } @@ -2339,10 +2336,9 @@ class StoreSDNode : public LSBaseSDNode { friend class SelectionDAG; StoreSDNode(SDValue *ChainValuePtrOff, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT SVT, - const Value *SV, int O=0, unsigned Align=0, bool Vol=false, - unsigned OAlign=0) + const Value *SV, int O=0, unsigned Align=0, bool Vol=false) : LSBaseSDNode(ISD::STORE, dl, ChainValuePtrOff, 4, - VTs, AM, SVT, SV, O, Align, Vol, OAlign) { + VTs, AM, SVT, SV, O, Align, Vol) { SubclassData |= (unsigned short)isTrunc; assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); } diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 33adf08f674..a03f825a9f0 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -170,8 +170,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { DAG.getUNDEF(N->getBasePtr().getValueType()), N->getSrcValue(), N->getSrcValueOffset(), N->getMemoryVT().getVectorElementType(), - N->isVolatile(), N->getAlignment(), - N->getOriginalAlignment()); + N->isVolatile(), N->getOriginalAlignment()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -361,8 +360,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), - N->isVolatile(), N->getAlignment(), - N->getOriginalAlignment()); + N->isVolatile(), N->getOriginalAlignment()); } @@ -716,23 +714,21 @@ void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, const Value *SV = LD->getSrcValue(); int SVOffset = LD->getSrcValueOffset(); EVT MemoryVT = LD->getMemoryVT(); - unsigned Alignment = LD->getAlignment(); - unsigned OrigAlignment = LD->getOriginalAlignment(); + unsigned Alignment = LD->getOriginalAlignment(); bool isVolatile = LD->isVolatile(); EVT LoMemVT, HiMemVT; GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, - SV, SVOffset, LoMemVT, isVolatile, Alignment, OrigAlignment); + SV, SVOffset, LoMemVT, isVolatile, Alignment); unsigned IncrementSize = LoMemVT.getSizeInBits()/8; Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, DAG.getIntPtrConstant(IncrementSize)); SVOffset += IncrementSize; - Alignment = MinAlign(Alignment, IncrementSize); Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, - SV, SVOffset, HiMemVT, isVolatile, Alignment, OrigAlignment); + SV, SVOffset, HiMemVT, isVolatile, Alignment); // Build a factor node to remember that this load is independent of the // other one. @@ -1081,8 +1077,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { SDValue Ptr = N->getBasePtr(); int SVOffset = N->getSrcValueOffset(); EVT MemoryVT = N->getMemoryVT(); - unsigned Alignment = N->getAlignment(); - unsigned OrigAlignment = N->getOriginalAlignment(); + unsigned Alignment = N->getOriginalAlignment(); bool isVol = N->isVolatile(); SDValue Lo, Hi; GetSplitVector(N->getOperand(1), Lo, Hi); @@ -1097,21 +1092,19 @@ SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { LoMemVT, isVol, Alignment); else Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, - isVol, Alignment, OrigAlignment); + isVol, Alignment); // Increment the pointer to the other half. Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, DAG.getIntPtrConstant(IncrementSize)); + SVOffset += IncrementSize; if (isTruncating) - Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, - N->getSrcValue(), SVOffset+IncrementSize, - HiMemVT, - isVol, MinAlign(Alignment, IncrementSize)); + Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, + HiMemVT, isVol, Alignment); else - Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, - isVol, MinAlign(Alignment, IncrementSize), - OrigAlignment); + Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, + isVol, Alignment); return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index fe45319339e..d0d45b5c77a 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -432,12 +432,14 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { const LoadSDNode *LD = cast(N); ID.AddInteger(LD->getMemoryVT().getRawBits()); ID.AddInteger(LD->getRawSubclassData()); + ID.AddInteger(LD->getSrcValueOffset()); break; } case ISD::STORE: { const StoreSDNode *ST = cast(N); ID.AddInteger(ST->getMemoryVT().getRawBits()); ID.AddInteger(ST->getRawSubclassData()); + ID.AddInteger(ST->getSrcValueOffset()); break; } case ISD::ATOMIC_CMP_SWAP: @@ -3646,12 +3648,9 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType, EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset, const Value *SV, int SVOffset, EVT EVT, - bool isVolatile, unsigned Alignment, - unsigned OrigAlignment) { + bool isVolatile, unsigned Alignment) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(VT); - if (OrigAlignment == 0) - OrigAlignment = Alignment; if (VT == EVT) { ExtType = ISD::NON_EXTLOAD; @@ -3682,12 +3681,13 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); ID.AddInteger(EVT.getRawBits()); ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, isVolatile, Alignment)); + ID.AddInteger(SVOffset); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); SDNode *N = NodeAllocator.Allocate(); new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, EVT, SV, SVOffset, - Alignment, isVolatile, OrigAlignment); + Alignment, isVolatile); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3696,11 +3696,10 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment, - unsigned OrigAlignment) { + bool isVolatile, unsigned Alignment) { SDValue Undef = getUNDEF(Ptr.getValueType()); return getLoad(ISD::UNINDEXED, dl, ISD::NON_EXTLOAD, VT, Chain, Ptr, Undef, - SV, SVOffset, VT, isVolatile, Alignment, OrigAlignment); + SV, SVOffset, VT, isVolatile, Alignment); } SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, @@ -3727,14 +3726,11 @@ SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment, - unsigned OrigAlignment) { + bool isVolatile, unsigned Alignment) { EVT VT = Val.getValueType(); if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(VT); - if (OrigAlignment == 0) - OrigAlignment = Alignment; SDVTList VTs = getVTList(MVT::Other); SDValue Undef = getUNDEF(Ptr.getValueType()); @@ -3744,12 +3740,13 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, ID.AddInteger(VT.getRawBits()); ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, isVolatile, Alignment)); + ID.AddInteger(SVOffset); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); SDNode *N = NodeAllocator.Allocate(); new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false, - VT, SV, SVOffset, Alignment, isVolatile, OrigAlignment); + VT, SV, SVOffset, Alignment, isVolatile); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3779,6 +3776,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, ID.AddInteger(SVT.getRawBits()); ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, isVolatile, Alignment)); + ID.AddInteger(SVOffset); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); @@ -4976,25 +4974,24 @@ GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, const GlobalValue *GA, MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, const Value *srcValue, int SVO, unsigned alignment, - bool vol, unsigned origAlign) - : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO), - OrigAlign(origAlign) { + bool vol) + : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) { SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment); assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!"); - assert(getAlignment() == alignment && "Alignment representation error!"); - assert(isVolatile() == vol && "Volatile representation error!"); + assert(getOriginalAlignment() == alignment && "Alignment encoding error!"); + assert(isVolatile() == vol && "Volatile encoding error!"); } MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, unsigned NumOps, EVT memvt, const Value *srcValue, int SVO, unsigned alignment, - bool vol, unsigned origAlign) + bool vol) : SDNode(Opc, dl, VTs, Ops, NumOps), - MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO), OrigAlign(origAlign) { + MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) { SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment); assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!"); - assert(getAlignment() == alignment && "Alignment representation error!"); - assert(isVolatile() == vol && "Volatile representation error!"); + assert(getOriginalAlignment() == alignment && "Alignment encoding error!"); + assert(isVolatile() == vol && "Volatile encoding error!"); } /// getMemOperand - Return a MachineMemOperand object describing the memory diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp index 8b9a5a5459d..c946bdad5d3 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp @@ -2815,7 +2815,7 @@ void SelectionDAGLowering::visitLoad(LoadInst &I) { DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, Ptr, DAG.getConstant(Offsets[i], PtrVT)), - SV, Offsets[i], isVolatile, Alignment, Alignment); + SV, Offsets[i], isVolatile, Alignment); Values[i] = L; Chains[i] = L.getValue(1); } @@ -2864,8 +2864,7 @@ void SelectionDAGLowering::visitStore(StoreInst &I) { DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, Ptr, DAG.getConstant(Offsets[i], PtrVT)), - PtrV, Offsets[i], - isVolatile, Alignment, Alignment); + PtrV, Offsets[i], isVolatile, Alignment); DAG.setRoot(DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other, &Chains[0], NumValues));