From e6798372ea38e5ea24c26282a0d69aa6e3829854 Mon Sep 17 00:00:00 2001 From: Nate Begeman Date: Tue, 15 Sep 2009 00:13:12 +0000 Subject: [PATCH] Add an "original alignment" field to load and store nodes. This enables the DAG Combiner to disambiguate chains for loads and stores of types which are broken up by the Legalizer into smaller pieces. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@81813 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/CodeGen/SelectionDAG.h | 12 +++---- include/llvm/CodeGen/SelectionDAGNodes.h | 31 ++++++++++------ lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 36 ++++++++++++------- .../SelectionDAG/SelectionDAGBuild.cpp | 11 +++--- 4 files changed, 54 insertions(+), 36 deletions(-) diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 9ab04151cb0..064fc874551 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -544,7 +544,7 @@ public: /// SDValue getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, bool isVolatile=false, - unsigned Alignment=0); + unsigned Alignment=0, unsigned OrigAlignment=0); SDValue getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, EVT EVT, bool isVolatile=false, @@ -552,16 +552,16 @@ public: SDValue getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM); SDValue getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType, - EVT VT, SDValue Chain, - SDValue Ptr, SDValue Offset, - const Value *SV, int SVOffset, EVT EVT, - bool isVolatile=false, unsigned Alignment=0); + EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset, + const Value *SV, int SVOffset, EVT EVT, + bool isVolatile=false, unsigned Alignment=0, + unsigned OrigAlignment=0); /// getStore - Helper function to build ISD::STORE nodes. /// SDValue getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, bool isVolatile=false, - unsigned Alignment=0); + unsigned Alignment=0, unsigned OrigAlignment=0); SDValue getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, EVT TVT, bool isVolatile=false, unsigned Alignment=0); diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index c42ff38505a..a3ce5d432d3 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1519,17 +1519,22 @@ private: //! SVOffset - Memory location offset. Note that base is defined in MemSDNode int SVOffset; + //! OrigAlign - The original alignment of this MemSDNode in the case where + // this node was created by legalize from a MemSDNode with known alignment. + unsigned OrigAlign; + public: MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT MemoryVT, const Value *srcValue, int SVOff, - unsigned alignment, bool isvolatile); + unsigned alignment, bool isvolatile, unsigned oalign); MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, unsigned NumOps, EVT MemoryVT, const Value *srcValue, int SVOff, - unsigned alignment, bool isvolatile); + unsigned alignment, bool isvolatile, unsigned oalign); /// Returns alignment and volatility of the memory access unsigned getAlignment() const { return (1u << (SubclassData >> 6)) >> 1; } + unsigned getOriginalAlignment() const { return OrigAlign; } bool isVolatile() const { return (SubclassData >> 5) & 1; } /// getRawSubclassData - Return the SubclassData value, which contains an @@ -1600,14 +1605,14 @@ public: SDValue Cmp, SDValue Swp, const Value* SrcVal, unsigned Align=0) : MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0, - Align, /*isVolatile=*/true) { + Align, /*isVolatile=*/true, /* OrigAlign=*/0) { InitOperands(Ops, Chain, Ptr, Cmp, Swp); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value* SrcVal, unsigned Align=0) : MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0, - Align, /*isVolatile=*/true) { + Align, /*isVolatile=*/true, /* OrigAlign=*/0) { InitOperands(Ops, Chain, Ptr, Val); } @@ -1648,7 +1653,8 @@ public: const SDValue *Ops, unsigned NumOps, EVT MemoryVT, const Value *srcValue, int SVO, unsigned Align, bool Vol, bool ReadMem, bool WriteMem) - : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol), + : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol, + /* OrigAlign=*/0), ReadMem(ReadMem), WriteMem(WriteMem) { } @@ -2263,8 +2269,9 @@ class LSBaseSDNode : public MemSDNode { public: LSBaseSDNode(ISD::NodeType NodeTy, DebugLoc dl, SDValue *Operands, unsigned numOperands, SDVTList VTs, ISD::MemIndexedMode AM, - EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol) - : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol) { + EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol, + unsigned OAlign) + : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol, OAlign) { assert(Align != 0 && "Loads and stores should have non-zero aligment"); SubclassData |= AM << 2; assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); @@ -2302,9 +2309,10 @@ class LoadSDNode : public LSBaseSDNode { friend class SelectionDAG; LoadSDNode(SDValue *ChainPtrOff, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT LVT, - const Value *SV, int O=0, unsigned Align=0, bool Vol=false) + const Value *SV, int O=0, unsigned Align=0, bool Vol=false, + unsigned OAlign=0) : LSBaseSDNode(ISD::LOAD, dl, ChainPtrOff, 3, - VTs, AM, LVT, SV, O, Align, Vol) { + VTs, AM, LVT, SV, O, Align, Vol, OAlign) { SubclassData |= (unsigned short)ETy; assert(getExtensionType() == ETy && "LoadExtType encoding error!"); } @@ -2331,9 +2339,10 @@ class StoreSDNode : public LSBaseSDNode { friend class SelectionDAG; StoreSDNode(SDValue *ChainValuePtrOff, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT SVT, - const Value *SV, int O=0, unsigned Align=0, bool Vol=false) + const Value *SV, int O=0, unsigned Align=0, bool Vol=false, + unsigned OAlign=0) : LSBaseSDNode(ISD::STORE, dl, ChainValuePtrOff, 4, - VTs, AM, SVT, SV, O, Align, Vol) { + VTs, AM, SVT, SV, O, Align, Vol, OAlign) { SubclassData |= (unsigned short)isTrunc; assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 0a523fa8dc2..014e62ef469 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -3646,9 +3646,12 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType, EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset, const Value *SV, int SVOffset, EVT EVT, - bool isVolatile, unsigned Alignment) { + bool isVolatile, unsigned Alignment, + unsigned OrigAlignment) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(VT); + if (OrigAlignment == 0) + OrigAlignment = Alignment; if (VT == EVT) { ExtType = ISD::NON_EXTLOAD; @@ -3679,12 +3682,13 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); ID.AddInteger(EVT.getRawBits()); ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, isVolatile, Alignment)); + ID.AddInteger(OrigAlignment); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); SDNode *N = NodeAllocator.Allocate(); new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, EVT, SV, SVOffset, - Alignment, isVolatile); + Alignment, isVolatile, OrigAlignment); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3693,10 +3697,11 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment) { + bool isVolatile, unsigned Alignment, + unsigned OrigAlignment) { SDValue Undef = getUNDEF(Ptr.getValueType()); return getLoad(ISD::UNINDEXED, dl, ISD::NON_EXTLOAD, VT, Chain, Ptr, Undef, - SV, SVOffset, VT, isVolatile, Alignment); + SV, SVOffset, VT, isVolatile, Alignment, OrigAlignment); } SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, @@ -3723,11 +3728,14 @@ SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment) { + bool isVolatile, unsigned Alignment, + unsigned OrigAlignment) { EVT VT = Val.getValueType(); if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(VT); + if (OrigAlignment == 0) + OrigAlignment = Alignment; SDVTList VTs = getVTList(MVT::Other); SDValue Undef = getUNDEF(Ptr.getValueType()); @@ -3737,12 +3745,13 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, ID.AddInteger(VT.getRawBits()); ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, isVolatile, Alignment)); + ID.AddInteger(OrigAlignment); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); SDNode *N = NodeAllocator.Allocate(); new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false, - VT, SV, SVOffset, Alignment, isVolatile); + VT, SV, SVOffset, Alignment, isVolatile, OrigAlignment); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4968,9 +4977,10 @@ GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, const GlobalValue *GA, } MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, - const Value *srcValue, int SVO, - unsigned alignment, bool vol) - : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) { + const Value *srcValue, int SVO, unsigned alignment, + bool vol, unsigned origAlign) + : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO), + OrigAlign(origAlign) { SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment); assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!"); assert(getAlignment() == alignment && "Alignment representation error!"); @@ -4978,11 +4988,11 @@ MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, } MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, - const SDValue *Ops, - unsigned NumOps, EVT memvt, const Value *srcValue, - int SVO, unsigned alignment, bool vol) + const SDValue *Ops, unsigned NumOps, EVT memvt, + const Value *srcValue, int SVO, unsigned alignment, + bool vol, unsigned origAlign) : SDNode(Opc, dl, VTs, Ops, NumOps), - MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) { + MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO), OrigAlign(origAlign) { SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment); assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!"); assert(getAlignment() == alignment && "Alignment representation error!"); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp index ad599124fb8..8b9a5a5459d 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp @@ -2812,11 +2812,10 @@ void SelectionDAGLowering::visitLoad(LoadInst &I) { EVT PtrVT = Ptr.getValueType(); for (unsigned i = 0; i != NumValues; ++i) { SDValue L = DAG.getLoad(ValueVTs[i], getCurDebugLoc(), Root, - DAG.getNode(ISD::ADD, getCurDebugLoc(), - PtrVT, Ptr, - DAG.getConstant(Offsets[i], PtrVT)), - SV, Offsets[i], - isVolatile, Alignment); + DAG.getNode(ISD::ADD, getCurDebugLoc(), + PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)), + SV, Offsets[i], isVolatile, Alignment, Alignment); Values[i] = L; Chains[i] = L.getValue(1); } @@ -2866,7 +2865,7 @@ void SelectionDAGLowering::visitStore(StoreInst &I) { PtrVT, Ptr, DAG.getConstant(Offsets[i], PtrVT)), PtrV, Offsets[i], - isVolatile, Alignment); + isVolatile, Alignment, Alignment); DAG.setRoot(DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other, &Chains[0], NumValues)); -- 2.34.1