From d63390cba15f10600d550201f2e9109e75933a0f Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Tue, 15 Apr 2014 07:22:52 +0000 Subject: [PATCH] Break PseudoSourceValue out of the Value hierarchy. It is now the root of its own tree containing FixedStackPseudoSourceValue (which you can use isa/dyn_cast on) and MipsCallEntry (which you can't). Anything that needs to use either a PseudoSourceValue* and Value* is strongly encouraged to use a MachinePointerInfo instead. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@206255 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/CodeGen/MachineMemOperand.h | 25 ++- include/llvm/CodeGen/PseudoSourceValue.h | 29 ++-- include/llvm/CodeGen/SelectionDAG.h | 11 +- include/llvm/CodeGen/SelectionDAGNodes.h | 3 +- include/llvm/IR/Value.h | 3 - lib/CodeGen/MachineFunction.cpp | 8 +- lib/CodeGen/MachineInstr.cpp | 27 ++-- lib/CodeGen/MachineLICM.cpp | 11 +- lib/CodeGen/PseudoSourceValue.cpp | 10 +- lib/CodeGen/ScheduleDAGInstrs.cpp | 78 ++++----- lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 149 +++++------------- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 37 +---- .../SelectionDAG/SelectionDAGBuilder.cpp | 18 ++- lib/CodeGen/StackColoring.cpp | 9 +- lib/CodeGen/StackSlotColoring.cpp | 15 +- lib/CodeGen/TargetInstrInfo.cpp | 12 +- lib/IR/AsmWriter.cpp | 6 - lib/Target/AArch64/AArch64ISelLowering.cpp | 2 +- lib/Target/Mips/MipsDelaySlotFiller.cpp | 48 +++--- lib/Target/Mips/MipsISelLowering.cpp | 4 +- lib/Target/Mips/MipsOptimizePICCall.cpp | 28 ++-- lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 10 +- lib/Target/PowerPC/PPCISelLowering.cpp | 4 +- lib/Target/R600/AMDGPUISelDAGToDAG.cpp | 57 ++++--- lib/Target/R600/AMDGPUISelLowering.cpp | 3 +- lib/Target/R600/R600ISelLowering.cpp | 4 +- lib/Target/SystemZ/SystemZISelDAGToDAG.cpp | 4 +- lib/Target/X86/X86ISelLowering.cpp | 3 +- 28 files changed, 280 insertions(+), 338 deletions(-) diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h index 68b749a56f1..2532c16271f 100644 --- a/include/llvm/CodeGen/MachineMemOperand.h +++ b/include/llvm/CodeGen/MachineMemOperand.h @@ -16,11 +16,13 @@ #ifndef LLVM_CODEGEN_MACHINEMEMOPERAND_H #define LLVM_CODEGEN_MACHINEMEMOPERAND_H +#include "llvm/ADT/PointerUnion.h" +#include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/IR/Value.h" // PointerLikeTypeTraits #include "llvm/Support/DataTypes.h" namespace llvm { -class Value; class FoldingSetNodeID; class MDNode; class raw_ostream; @@ -33,7 +35,7 @@ struct MachinePointerInfo { /// V - This is the IR pointer value for the access, or it is null if unknown. /// If this is null, then the access is to a pointer in the default address /// space. - const Value *V; + PointerUnion V; /// Offset - This is an offset from the base Value*. int64_t Offset; @@ -41,9 +43,15 @@ struct MachinePointerInfo { explicit MachinePointerInfo(const Value *v = nullptr, int64_t offset = 0) : V(v), Offset(offset) {} + explicit MachinePointerInfo(const PseudoSourceValue *v, + int64_t offset = 0) + : V(v), Offset(offset) {} + MachinePointerInfo getWithOffset(int64_t O) const { - if (V == nullptr) return MachinePointerInfo(nullptr, 0); - return MachinePointerInfo(V, Offset+O); + if (V.isNull()) return MachinePointerInfo(); + if (V.is()) + return MachinePointerInfo(V.get(), Offset+O); + return MachinePointerInfo(V.get(), Offset+O); } /// getAddrSpace - Return the LLVM IR address space number that this pointer @@ -121,7 +129,13 @@ public: /// other PseudoSourceValue member functions which return objects which stand /// for frame/stack pointer relative references and other special references /// which are not representable in the high-level IR. - const Value *getValue() const { return PtrInfo.V; } + const Value *getValue() const { return PtrInfo.V.dyn_cast(); } + + const PseudoSourceValue *getPseudoValue() const { + return PtrInfo.V.dyn_cast(); + } + + const void *getOpaqueValue() const { return PtrInfo.V.getOpaqueValue(); } /// getFlags - Return the raw flags of the source value, \see MemOperandFlags. unsigned int getFlags() const { return Flags & ((1 << MOMaxBits) - 1); } @@ -177,6 +191,7 @@ public: /// should only be used when an object is being relocated and all references /// to it are being updated. void setValue(const Value *NewSV) { PtrInfo.V = NewSV; } + void setValue(const PseudoSourceValue *NewSV) { PtrInfo.V = NewSV; } void setOffset(int64_t NewOffset) { PtrInfo.Offset = NewOffset; } /// Profile - Gather unique data for the object. diff --git a/include/llvm/CodeGen/PseudoSourceValue.h b/include/llvm/CodeGen/PseudoSourceValue.h index 0af89155519..0f35827e236 100644 --- a/include/llvm/CodeGen/PseudoSourceValue.h +++ b/include/llvm/CodeGen/PseudoSourceValue.h @@ -18,21 +18,30 @@ namespace llvm { class MachineFrameInfo; + class MachineMemOperand; class raw_ostream; /// PseudoSourceValue - Special value supplied for machine level alias /// analysis. It indicates that a memory access references the functions /// stack frame (e.g., a spill slot), below the stack frame (e.g., argument /// space), or constant pool. - class PseudoSourceValue : public Value { + class PseudoSourceValue { private: + friend raw_ostream &llvm::operator<<(raw_ostream &OS, + const MachineMemOperand &MMO); + /// printCustom - Implement printing for PseudoSourceValue. This is called /// from Value::print or Value's operator<<. /// - void printCustom(raw_ostream &O) const override; + virtual void printCustom(raw_ostream &O) const; public: - explicit PseudoSourceValue(enum ValueTy Subclass = PseudoSourceValueVal); + /// isFixed - Whether this is a FixedStackPseudoSourceValue. + bool isFixed; + + explicit PseudoSourceValue(bool isFixed = false); + + virtual ~PseudoSourceValue(); /// isConstant - Test whether the memory pointed to by this /// PseudoSourceValue has a constant value. @@ -47,14 +56,6 @@ namespace llvm { /// PseudoSourceValue can ever alias an LLVM IR Value. virtual bool mayAlias(const MachineFrameInfo *) const; - /// classof - Methods for support type inquiry through isa, cast, and - /// dyn_cast: - /// - static inline bool classof(const Value *V) { - return V->getValueID() == PseudoSourceValueVal || - V->getValueID() == FixedStackPseudoSourceValueVal; - } - /// A pseudo source value referencing a fixed stack frame entry, /// e.g., a spill slot. static const PseudoSourceValue *getFixedStack(int FI); @@ -84,13 +85,13 @@ namespace llvm { const int FI; public: explicit FixedStackPseudoSourceValue(int fi) : - PseudoSourceValue(FixedStackPseudoSourceValueVal), FI(fi) {} + PseudoSourceValue(true), FI(fi) {} /// classof - Methods for support type inquiry through isa, cast, and /// dyn_cast: /// - static inline bool classof(const Value *V) { - return V->getValueID() == FixedStackPseudoSourceValueVal; + static inline bool classof(const PseudoSourceValue *V) { + return V->isFixed == true; } bool isConstant(const MachineFrameInfo *MFI) const override; diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index b1e63f29a22..16d1864b60e 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -705,7 +705,7 @@ public: /// getAtomic - Gets a node for an atomic op, produces result (if relevant) /// and chain and takes 2 operands. SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, - SDValue Ptr, SDValue Val, const Value* PtrVal, + SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, @@ -715,11 +715,6 @@ public: /// getAtomic - Gets a node for an atomic op, produces result and chain and /// takes 1 operand. - SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT, - SDValue Chain, SDValue Ptr, const Value* PtrVal, - unsigned Alignment, - AtomicOrdering Ordering, - SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, AtomicOrdering Ordering, @@ -1167,6 +1162,8 @@ public: void ExtractVectorElements(SDValue Op, SmallVectorImpl &Args, unsigned Start = 0, unsigned Count = 0); + unsigned getEVTAlignment(EVT MemoryVT) const; + private: bool RemoveNodeFromCSEMaps(SDNode *N); void AddModifiedNodeToCSEMaps(SDNode *N); @@ -1180,8 +1177,6 @@ private: void DeleteNodeNotInCSEMaps(SDNode *N); void DeallocateNode(SDNode *N); - unsigned getEVTAlignment(EVT MemoryVT) const; - void allnodes_clear(); /// VTList - List of non-single value types. diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 99ff5cb64c3..fae99b5f744 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1032,8 +1032,7 @@ public: return SynchronizationScope((SubclassData >> 12) & 1); } - /// Returns the SrcValue and offset that describes the location of the access - const Value *getSrcValue() const { return MMO->getValue(); } + // Returns the offset from the location of the access. int64_t getSrcValueOffset() const { return MMO->getOffset(); } /// Returns the TBAAInfo that describes the dereference. diff --git a/include/llvm/IR/Value.h b/include/llvm/IR/Value.h index 9d69a432520..d50525f6c33 100644 --- a/include/llvm/IR/Value.h +++ b/include/llvm/IR/Value.h @@ -328,9 +328,6 @@ public: MDNodeVal, // This is an instance of MDNode MDStringVal, // This is an instance of MDString InlineAsmVal, // This is an instance of InlineAsm - PseudoSourceValueVal, // This is an instance of PseudoSourceValue - FixedStackPseudoSourceValueVal, // This is an instance of - // FixedStackPseudoSourceValue InstructionVal, // This is an instance of Instruction // Enum values starting at InstructionVal are used for Instructions; // don't add new values here! diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index 13383847848..68c12617c60 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -236,8 +236,14 @@ MachineFunction::getMachineMemOperand(MachinePointerInfo PtrInfo, unsigned f, MachineMemOperand * MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, int64_t Offset, uint64_t Size) { + if (MMO->getValue()) + return new (Allocator) + MachineMemOperand(MachinePointerInfo(MMO->getValue(), + MMO->getOffset()+Offset), + MMO->getFlags(), Size, + MMO->getBaseAlignment(), nullptr); return new (Allocator) - MachineMemOperand(MachinePointerInfo(MMO->getValue(), + MachineMemOperand(MachinePointerInfo(MMO->getPseudoValue(), MMO->getOffset()+Offset), MMO->getFlags(), Size, MMO->getBaseAlignment(), nullptr); diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index aa6de624de5..f477e91dfbc 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -399,8 +399,8 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const { /// getAddrSpace - Return the LLVM IR address space number that this pointer /// points into. unsigned MachinePointerInfo::getAddrSpace() const { - if (!V) return 0; - return cast(V->getType())->getAddressSpace(); + if (V.isNull() || V.is()) return 0; + return cast(V.get()->getType())->getAddressSpace(); } /// getConstantPool - Return a MachinePointerInfo record that refers to the @@ -434,7 +434,8 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, unsigned f, : PtrInfo(ptrinfo), Size(s), Flags((f & ((1 << MOMaxBits) - 1)) | ((Log2_32(a) + 1) << MOMaxBits)), TBAAInfo(TBAAInfo), Ranges(Ranges) { - assert((!PtrInfo.V || isa(PtrInfo.V->getType())) && + assert((PtrInfo.V.isNull() || PtrInfo.V.is() || + isa(PtrInfo.V.get()->getType())) && "invalid pointer value"); assert(getBaseAlignment() == a && "Alignment is not a power of 2!"); assert((isLoad() || isStore()) && "Not a load/store!"); @@ -445,7 +446,7 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, unsigned f, void MachineMemOperand::Profile(FoldingSetNodeID &ID) const { ID.AddInteger(getOffset()); ID.AddInteger(Size); - ID.AddPointer(getValue()); + ID.AddPointer(getOpaqueValue()); ID.AddInteger(Flags); } @@ -486,10 +487,12 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) { // Print the address information. OS << "["; - if (!MMO.getValue()) - OS << ""; + if (const Value *V = MMO.getValue()) + V->printAsOperand(OS, /*PrintType=*/false); + else if (const PseudoSourceValue *PSV = MMO.getPseudoValue()) + PSV->printCustom(OS); else - MMO.getValue()->printAsOperand(OS, /*PrintType=*/false); + OS << ""; unsigned AS = MMO.getAddrSpace(); if (AS != 0) @@ -1366,11 +1369,13 @@ bool MachineInstr::isInvariantLoad(AliasAnalysis *AA) const { if ((*I)->isStore()) return false; if ((*I)->isInvariant()) return true; + + // A load from a constant PseudoSourceValue is invariant. + if (const PseudoSourceValue *PSV = (*I)->getPseudoValue()) + if (PSV->isConstant(MFI)) + continue; + if (const Value *V = (*I)->getValue()) { - // A load from a constant PseudoSourceValue is invariant. - if (const PseudoSourceValue *PSV = dyn_cast(V)) - if (PSV->isConstant(MFI)) - continue; // If we have an AliasAnalysis, ask it whether the memory is constant. if (AA && AA->pointsToConstantMemory( AliasAnalysis::Location(V, (*I)->getSize(), diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index 32379b6d824..8960d38e610 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -390,10 +390,10 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) { static bool InstructionStoresToFI(const MachineInstr *MI, int FI) { for (MachineInstr::mmo_iterator o = MI->memoperands_begin(), oe = MI->memoperands_end(); o != oe; ++o) { - if (!(*o)->isStore() || !(*o)->getValue()) + if (!(*o)->isStore() || !(*o)->getPseudoValue()) continue; if (const FixedStackPseudoSourceValue *Value = - dyn_cast((*o)->getValue())) { + dyn_cast((*o)->getPseudoValue())) { if (Value->getFrameIndex() == FI) return true; } @@ -882,10 +882,9 @@ static bool isLoadFromGOTOrConstantPool(MachineInstr &MI) { assert (MI.mayLoad() && "Expected MI that loads!"); for (MachineInstr::mmo_iterator I = MI.memoperands_begin(), E = MI.memoperands_end(); I != E; ++I) { - if (const Value *V = (*I)->getValue()) { - if (const PseudoSourceValue *PSV = dyn_cast(V)) - if (PSV == PSV->getGOT() || PSV == PSV->getConstantPool()) - return true; + if (const PseudoSourceValue *PSV = (*I)->getPseudoValue()) { + if (PSV == PSV->getGOT() || PSV == PSV->getConstantPool()) + return true; } } return false; diff --git a/lib/CodeGen/PseudoSourceValue.cpp b/lib/CodeGen/PseudoSourceValue.cpp index 85649111d7f..12b2c902f8d 100644 --- a/lib/CodeGen/PseudoSourceValue.cpp +++ b/lib/CodeGen/PseudoSourceValue.cpp @@ -58,13 +58,9 @@ static const char *const PSVNames[] = { "ConstantPool" }; -// FIXME: THIS IS A HACK!!!! -// Eventually these should be uniqued on LLVMContext rather than in a managed -// static. For now, we can safely use the global context for the time being to -// squeak by. -PseudoSourceValue::PseudoSourceValue(enum ValueTy Subclass) : - Value(Type::getInt8PtrTy(getGlobalContext()), - Subclass) {} +PseudoSourceValue::PseudoSourceValue(bool isFixed) : isFixed(isFixed) {} + +PseudoSourceValue::~PseudoSourceValue() {} void PseudoSourceValue::printCustom(raw_ostream &O) const { O << PSVNames[this - PSVGlobals->PSVs]; diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index d36322dc302..9c516c6edd9 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -98,7 +98,7 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) { /// and adds support for basic ptrtoint+arithmetic+inttoptr sequences. static void getUnderlyingObjects(const Value *V, SmallVectorImpl &Objects) { - SmallPtrSet Visited; + SmallPtrSet Visited; SmallVector Working(1, V); do { V = Working.pop_back_val(); @@ -124,7 +124,8 @@ static void getUnderlyingObjects(const Value *V, } while (!Working.empty()); } -typedef SmallVector, 4> +typedef PointerUnion ValueType; +typedef SmallVector, 4> UnderlyingObjectsVector; /// getUnderlyingObjectsForInstr - If this machine instr has memory reference @@ -134,25 +135,27 @@ static void getUnderlyingObjectsForInstr(const MachineInstr *MI, const MachineFrameInfo *MFI, UnderlyingObjectsVector &Objects) { if (!MI->hasOneMemOperand() || - !(*MI->memoperands_begin())->getValue() || + (!(*MI->memoperands_begin())->getValue() && + !(*MI->memoperands_begin())->getPseudoValue()) || (*MI->memoperands_begin())->isVolatile()) return; - const Value *V = (*MI->memoperands_begin())->getValue(); - if (!V) - return; - - if (const PseudoSourceValue *PSV = dyn_cast(V)) { + if (const PseudoSourceValue *PSV = + (*MI->memoperands_begin())->getPseudoValue()) { // For now, ignore PseudoSourceValues which may alias LLVM IR values // because the code that uses this function has no way to cope with // such aliases. if (!PSV->isAliased(MFI)) { bool MayAlias = PSV->mayAlias(MFI); - Objects.push_back(UnderlyingObjectsVector::value_type(V, MayAlias)); + Objects.push_back(UnderlyingObjectsVector::value_type(PSV, MayAlias)); } return; } + const Value *V = (*MI->memoperands_begin())->getValue(); + if (!V) + return; + SmallVector Objs; getUnderlyingObjects(V, Objs); @@ -160,8 +163,6 @@ static void getUnderlyingObjectsForInstr(const MachineInstr *MI, I != IE; ++I) { V = *I; - assert(!isa(V) && "Underlying value is a stack slot!"); - if (!isIdentifiedObject(V)) { Objects.clear(); return; @@ -477,6 +478,15 @@ static inline bool isUnsafeMemoryObject(MachineInstr *MI, if ((*MI->memoperands_begin())->isVolatile() || MI->hasUnmodeledSideEffects()) return true; + + if ((*MI->memoperands_begin())->getPseudoValue()) { + // Similarly to getUnderlyingObjectForInstr: + // For now, ignore PseudoSourceValues which may alias LLVM IR values + // because the code that uses this function has no way to cope with + // such aliases. + return true; + } + const Value *V = (*MI->memoperands_begin())->getValue(); if (!V) return true; @@ -485,19 +495,8 @@ static inline bool isUnsafeMemoryObject(MachineInstr *MI, getUnderlyingObjects(V, Objs); for (SmallVectorImpl::iterator I = Objs.begin(), IE = Objs.end(); I != IE; ++I) { - V = *I; - - if (const PseudoSourceValue *PSV = dyn_cast(V)) { - // Similarly to getUnderlyingObjectForInstr: - // For now, ignore PseudoSourceValues which may alias LLVM IR values - // because the code that uses this function has no way to cope with - // such aliases. - if (PSV->isAliased(MFI)) - return true; - } - // Does this pointer refer to a distinct and identifiable object? - if (!isIdentifiedObject(V)) + if (!isIdentifiedObject(*I)) return true; } @@ -535,6 +534,9 @@ static bool MIsNeedChainEdge(AliasAnalysis *AA, const MachineFrameInfo *MFI, MachineMemOperand *MMOa = *MIa->memoperands_begin(); MachineMemOperand *MMOb = *MIb->memoperands_begin(); + if (!MMOa->getValue() || !MMOb->getValue()) + return true; + // The following interface to AA is fashioned after DAGCombiner::isAlias // and operates with MachineMemOperand offset with some important // assumptions: @@ -751,8 +753,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, // so that they can be given more precise dependencies. We track // separately the known memory locations that may alias and those // that are known not to alias - MapVector > AliasMemDefs, NonAliasMemDefs; - MapVector > AliasMemUses, NonAliasMemUses; + MapVector > AliasMemDefs, NonAliasMemDefs; + MapVector > AliasMemUses, NonAliasMemUses; std::set RejectMemNodes; // Remove any stale debug info; sometimes BuildSchedGraph is called again @@ -848,13 +850,13 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, if (isGlobalMemoryObject(AA, MI)) { // Be conservative with these and add dependencies on all memory // references, even those that are known to not alias. - for (MapVector >::iterator I = + for (MapVector >::iterator I = NonAliasMemDefs.begin(), E = NonAliasMemDefs.end(); I != E; ++I) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) { I->second[i]->addPred(SDep(SU, SDep::Barrier)); } } - for (MapVector >::iterator I = + for (MapVector >::iterator I = NonAliasMemUses.begin(), E = NonAliasMemUses.end(); I != E; ++I) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) { SDep Dep(SU, SDep::Barrier); @@ -888,12 +890,12 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) addChainDependency(AAForDep, MFI, SU, PendingLoads[k], RejectMemNodes, TrueMemOrderLatency); - for (MapVector >::iterator I = + for (MapVector >::iterator I = AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) addChainDependency(AAForDep, MFI, SU, I->second[i], RejectMemNodes); } - for (MapVector >::iterator I = + for (MapVector >::iterator I = AliasMemUses.begin(), E = AliasMemUses.end(); I != E; ++I) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) addChainDependency(AAForDep, MFI, SU, I->second[i], RejectMemNodes, @@ -916,7 +918,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, bool MayAlias = false; for (UnderlyingObjectsVector::iterator K = Objs.begin(), KE = Objs.end(); K != KE; ++K) { - const Value *V = K->getPointer(); + ValueType V = K->getPointer(); bool ThisMayAlias = K->getInt(); if (ThisMayAlias) MayAlias = true; @@ -924,9 +926,9 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, // A store to a specific PseudoSourceValue. Add precise dependencies. // Record the def in MemDefs, first adding a dep if there is // an existing def. - MapVector >::iterator I = + MapVector >::iterator I = ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V)); - MapVector >::iterator IE = + MapVector >::iterator IE = ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); if (I != IE) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) @@ -949,9 +951,9 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, } } // Handle the uses in MemUses, if there are any. - MapVector >::iterator J = + MapVector >::iterator J = ((ThisMayAlias) ? AliasMemUses.find(V) : NonAliasMemUses.find(V)); - MapVector >::iterator JE = + MapVector >::iterator JE = ((ThisMayAlias) ? AliasMemUses.end() : NonAliasMemUses.end()); if (J != JE) { for (unsigned i = 0, e = J->second.size(); i != e; ++i) @@ -996,7 +998,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, if (Objs.empty()) { // A load with no underlying object. Depend on all // potentially aliasing stores. - for (MapVector >::iterator I = + for (MapVector >::iterator I = AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) for (unsigned i = 0, e = I->second.size(); i != e; ++i) addChainDependency(AAForDep, MFI, SU, I->second[i], @@ -1010,16 +1012,16 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, for (UnderlyingObjectsVector::iterator J = Objs.begin(), JE = Objs.end(); J != JE; ++J) { - const Value *V = J->getPointer(); + ValueType V = J->getPointer(); bool ThisMayAlias = J->getInt(); if (ThisMayAlias) MayAlias = true; // A load from a specific PseudoSourceValue. Add precise dependencies. - MapVector >::iterator I = + MapVector >::iterator I = ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V)); - MapVector >::iterator IE = + MapVector >::iterator IE = ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); if (I != IE) for (unsigned i = 0, e = I->second.size(); i != e; ++i) diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 2e2bfee2bc2..f0c604de180 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -317,26 +317,7 @@ namespace { /// isAlias - Return true if there is any possibility that the two addresses /// overlap. - bool isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, - const Value *SrcValue1, int SrcValueOffset1, - unsigned SrcValueAlign1, - const MDNode *TBAAInfo1, - SDValue Ptr2, int64_t Size2, bool IsVolatile2, - const Value *SrcValue2, int SrcValueOffset2, - unsigned SrcValueAlign2, - const MDNode *TBAAInfo2) const; - - /// isAlias - Return true if there is any possibility that the two addresses - /// overlap. - bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1); - - /// FindAliasInfo - Extracts the relevant alias information from the memory - /// node. Returns true if the operand was a load. - bool FindAliasInfo(SDNode *N, - SDValue &Ptr, int64_t &Size, bool &IsVolatile, - const Value *&SrcValue, int &SrcValueOffset, - unsigned &SrcValueAlignment, - const MDNode *&TBAAInfo) const; + bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const; /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, /// looking for a better chain (aliasing node.) @@ -11293,31 +11274,27 @@ static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset, /// isAlias - Return true if there is any possibility that the two addresses /// overlap. -bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, - const Value *SrcValue1, int SrcValueOffset1, - unsigned SrcValueAlign1, - const MDNode *TBAAInfo1, - SDValue Ptr2, int64_t Size2, bool IsVolatile2, - const Value *SrcValue2, int SrcValueOffset2, - unsigned SrcValueAlign2, - const MDNode *TBAAInfo2) const { +bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const { // If they are the same then they must be aliases. - if (Ptr1 == Ptr2) return true; + if (Op0->getBasePtr() == Op1->getBasePtr()) return true; // If they are both volatile then they cannot be reordered. - if (IsVolatile1 && IsVolatile2) return true; + if (Op0->isVolatile() && Op1->isVolatile()) return true; // Gather base node and offset information. SDValue Base1, Base2; int64_t Offset1, Offset2; const GlobalValue *GV1, *GV2; const void *CV1, *CV2; - bool isFrameIndex1 = FindBaseOffset(Ptr1, Base1, Offset1, GV1, CV1); - bool isFrameIndex2 = FindBaseOffset(Ptr2, Base2, Offset2, GV2, CV2); + bool isFrameIndex1 = FindBaseOffset(Op0->getBasePtr(), + Base1, Offset1, GV1, CV1); + bool isFrameIndex2 = FindBaseOffset(Op1->getBasePtr(), + Base2, Offset2, GV2, CV2); // If they have a same base address then check to see if they overlap. if (Base1 == Base2 || (GV1 && (GV1 == GV2)) || (CV1 && (CV1 == CV2))) - return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1); + return !((Offset1 + (Op0->getMemoryVT().getSizeInBits() >> 3)) <= Offset2 || + (Offset2 + (Op1->getMemoryVT().getSizeInBits() >> 3)) <= Offset1); // It is possible for different frame indices to alias each other, mostly // when tail call optimization reuses return address slots for arguments. @@ -11327,7 +11304,8 @@ bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); Offset1 += MFI->getObjectOffset(cast(Base1)->getIndex()); Offset2 += MFI->getObjectOffset(cast(Base2)->getIndex()); - return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1); + return !((Offset1 + (Op0->getMemoryVT().getSizeInBits() >> 3)) <= Offset2 || + (Offset2 + (Op1->getMemoryVT().getSizeInBits() >> 3)) <= Offset1); } // Otherwise, if we know what the bases are, and they aren't identical, then @@ -11339,15 +11317,18 @@ bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, // compared to the size and offset of the access, we may be able to prove they // do not alias. This check is conservative for now to catch cases created by // splitting vector types. - if ((SrcValueAlign1 == SrcValueAlign2) && - (SrcValueOffset1 != SrcValueOffset2) && - (Size1 == Size2) && (SrcValueAlign1 > Size1)) { - int64_t OffAlign1 = SrcValueOffset1 % SrcValueAlign1; - int64_t OffAlign2 = SrcValueOffset2 % SrcValueAlign1; + if ((Op0->getOriginalAlignment() == Op1->getOriginalAlignment()) && + (Op0->getSrcValueOffset() != Op1->getSrcValueOffset()) && + (Op0->getMemoryVT().getSizeInBits() >> 3 == + Op1->getMemoryVT().getSizeInBits() >> 3) && + (Op0->getOriginalAlignment() > Op0->getMemoryVT().getSizeInBits()) >> 3) { + int64_t OffAlign1 = Op0->getSrcValueOffset() % Op0->getOriginalAlignment(); + int64_t OffAlign2 = Op1->getSrcValueOffset() % Op1->getOriginalAlignment(); // There is no overlap between these relatively aligned accesses of similar // size, return no alias. - if ((OffAlign1 + Size1) <= OffAlign2 || (OffAlign2 + Size2) <= OffAlign1) + if ((OffAlign1 + (Op0->getMemoryVT().getSizeInBits() >> 3)) <= OffAlign2 || + (OffAlign2 + (Op1->getMemoryVT().getSizeInBits() >> 3)) <= OffAlign1) return false; } @@ -11358,16 +11339,22 @@ bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, CombinerAAOnlyFunc != DAG.getMachineFunction().getName()) UseAA = false; #endif - if (UseAA && SrcValue1 && SrcValue2) { + if (UseAA && + Op0->getMemOperand()->getValue() && Op1->getMemOperand()->getValue()) { // Use alias analysis information. - int64_t MinOffset = std::min(SrcValueOffset1, SrcValueOffset2); - int64_t Overlap1 = Size1 + SrcValueOffset1 - MinOffset; - int64_t Overlap2 = Size2 + SrcValueOffset2 - MinOffset; + int64_t MinOffset = std::min(Op0->getSrcValueOffset(), + Op1->getSrcValueOffset()); + int64_t Overlap1 = (Op0->getMemoryVT().getSizeInBits() >> 3) + + Op0->getSrcValueOffset() - MinOffset; + int64_t Overlap2 = (Op1->getMemoryVT().getSizeInBits() >> 3) + + Op1->getSrcValueOffset() - MinOffset; AliasAnalysis::AliasResult AAResult = - AA.alias(AliasAnalysis::Location(SrcValue1, Overlap1, - UseTBAA ? TBAAInfo1 : nullptr), - AliasAnalysis::Location(SrcValue2, Overlap2, - UseTBAA ? TBAAInfo2 : nullptr)); + AA.alias(AliasAnalysis::Location(Op0->getMemOperand()->getValue(), + Overlap1, + UseTBAA ? Op0->getTBAAInfo() : nullptr), + AliasAnalysis::Location(Op1->getMemOperand()->getValue(), + Overlap2, + UseTBAA ? Op1->getTBAAInfo() : nullptr)); if (AAResult == AliasAnalysis::NoAlias) return false; } @@ -11376,44 +11363,6 @@ bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, return true; } -bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) { - SDValue Ptr0, Ptr1; - int64_t Size0, Size1; - bool IsVolatile0, IsVolatile1; - const Value *SrcValue0, *SrcValue1; - int SrcValueOffset0, SrcValueOffset1; - unsigned SrcValueAlign0, SrcValueAlign1; - const MDNode *SrcTBAAInfo0, *SrcTBAAInfo1; - FindAliasInfo(Op0, Ptr0, Size0, IsVolatile0, SrcValue0, SrcValueOffset0, - SrcValueAlign0, SrcTBAAInfo0); - FindAliasInfo(Op1, Ptr1, Size1, IsVolatile1, SrcValue1, SrcValueOffset1, - SrcValueAlign1, SrcTBAAInfo1); - return isAlias(Ptr0, Size0, IsVolatile0, SrcValue0, SrcValueOffset0, - SrcValueAlign0, SrcTBAAInfo0, - Ptr1, Size1, IsVolatile1, SrcValue1, SrcValueOffset1, - SrcValueAlign1, SrcTBAAInfo1); -} - -/// FindAliasInfo - Extracts the relevant alias information from the memory -/// node. Returns true if the operand was a nonvolatile load. -bool DAGCombiner::FindAliasInfo(SDNode *N, - SDValue &Ptr, int64_t &Size, bool &IsVolatile, - const Value *&SrcValue, - int &SrcValueOffset, - unsigned &SrcValueAlign, - const MDNode *&TBAAInfo) const { - LSBaseSDNode *LS = cast(N); - - Ptr = LS->getBasePtr(); - Size = LS->getMemoryVT().getSizeInBits() >> 3; - IsVolatile = LS->isVolatile(); - SrcValue = LS->getSrcValue(); - SrcValueOffset = LS->getSrcValueOffset(); - SrcValueAlign = LS->getOriginalAlignment(); - TBAAInfo = LS->getTBAAInfo(); - return isa(LS) && !IsVolatile; -} - /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes, /// looking for aliasing nodes and adding them to the Aliases vector. void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain, @@ -11422,15 +11371,7 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain, SmallPtrSet Visited; // Visited node set. // Get alias information for node. - SDValue Ptr; - int64_t Size; - bool IsVolatile; - const Value *SrcValue; - int SrcValueOffset; - unsigned SrcValueAlign; - const MDNode *SrcTBAAInfo; - bool IsLoad = FindAliasInfo(N, Ptr, Size, IsVolatile, SrcValue, - SrcValueOffset, SrcValueAlign, SrcTBAAInfo); + bool IsLoad = isa(N) && !cast(N)->isVolatile(); // Starting off. Chains.push_back(OriginalChain); @@ -11469,24 +11410,12 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain, case ISD::LOAD: case ISD::STORE: { // Get alias information for Chain. - SDValue OpPtr; - int64_t OpSize; - bool OpIsVolatile; - const Value *OpSrcValue; - int OpSrcValueOffset; - unsigned OpSrcValueAlign; - const MDNode *OpSrcTBAAInfo; - bool IsOpLoad = FindAliasInfo(Chain.getNode(), OpPtr, OpSize, - OpIsVolatile, OpSrcValue, OpSrcValueOffset, - OpSrcValueAlign, - OpSrcTBAAInfo); + bool IsOpLoad = isa(Chain.getNode()) && + !cast(Chain.getNode())->isVolatile(); // If chain is alias then stop here. if (!(IsLoad && IsOpLoad) && - isAlias(Ptr, Size, IsVolatile, SrcValue, SrcValueOffset, - SrcValueAlign, SrcTBAAInfo, - OpPtr, OpSize, OpIsVolatile, OpSrcValue, OpSrcValueOffset, - OpSrcValueAlign, OpSrcTBAAInfo)) { + isAlias(cast(N), cast(Chain.getNode()))) { Aliases.push_back(Chain); } else { // Look further up the chain. diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 4bdf559cc78..5f43d818dc7 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -4386,37 +4386,6 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, return getAtomic(Opcode, dl, MemVT, VTs, Ops, 3, MMO, Ordering, SynchScope); } -SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, - EVT VT, SDValue Chain, - SDValue Ptr, - const Value* PtrVal, - unsigned Alignment, - AtomicOrdering Ordering, - SynchronizationScope SynchScope) { - if (Alignment == 0) // Ensure that codegen never sees alignment 0 - Alignment = getEVTAlignment(MemVT); - - MachineFunction &MF = getMachineFunction(); - // An atomic store does not load. An atomic load does not store. - // (An atomicrmw obviously both loads and stores.) - // For now, atomics are considered to be volatile always, and they are - // chained as such. - // FIXME: Volatile isn't really correct; we should keep track of atomic - // orderings in the memoperand. - unsigned Flags = MachineMemOperand::MOVolatile; - if (Opcode != ISD::ATOMIC_STORE) - Flags |= MachineMemOperand::MOLoad; - if (Opcode != ISD::ATOMIC_LOAD) - Flags |= MachineMemOperand::MOStore; - - MachineMemOperand *MMO = - MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, - MemVT.getStoreSize(), Alignment); - - return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, - Ordering, SynchScope); -} - SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, @@ -4574,7 +4543,7 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, // If we don't have a PtrInfo, infer the trivial frame index case to simplify // clients. - if (PtrInfo.V == nullptr) + if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(Ptr, Offset); MachineFunction &MF = getMachineFunction(); @@ -4701,7 +4670,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, if (isNonTemporal) Flags |= MachineMemOperand::MONonTemporal; - if (PtrInfo.V == nullptr) + if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(Ptr); MachineFunction &MF = getMachineFunction(); @@ -4756,7 +4725,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, if (isNonTemporal) Flags |= MachineMemOperand::MONonTemporal; - if (PtrInfo.V == nullptr) + if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(Ptr); MachineFunction &MF = getMachineFunction(); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 814047426ed..f09d010960b 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3696,13 +3696,21 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { if (I.getAlignment() < VT.getSizeInBits() / 8) report_fatal_error("Cannot generate unaligned atomic load"); + MachineMemOperand *MMO = + DAG.getMachineFunction(). + getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), + MachineMemOperand::MOVolatile | + MachineMemOperand::MOLoad, + VT.getStoreSize(), + I.getAlignment() ? I.getAlignment() : + DAG.getEVTAlignment(VT)); + InChain = TLI->prepareVolatileOrAtomicLoad(InChain, dl, DAG); SDValue L = - DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, - getValue(I.getPointerOperand()), - I.getPointerOperand(), I.getAlignment(), - TLI->getInsertFencesForAtomic() ? Monotonic : Order, - Scope); + DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, + getValue(I.getPointerOperand()), MMO, + TLI->getInsertFencesForAtomic() ? Monotonic : Order, + Scope); SDValue OutChain = L.getValue(1); diff --git a/lib/CodeGen/StackColoring.cpp b/lib/CodeGen/StackColoring.cpp index ac1f320b8b2..abe3db4b44e 100644 --- a/lib/CodeGen/StackColoring.cpp +++ b/lib/CodeGen/StackColoring.cpp @@ -509,11 +509,6 @@ void StackColoring::remapInstructions(DenseMap &SlotRemap) { // Update the MachineMemOperand to use the new alloca. for (MachineMemOperand *MMO : I.memoperands()) { - const Value *V = MMO->getValue(); - - if (!V) - continue; - // FIXME: In order to enable the use of TBAA when using AA in CodeGen, // we'll also need to update the TBAA nodes in MMOs with values // derived from the merged allocas. When doing this, we'll need to use @@ -523,10 +518,10 @@ void StackColoring::remapInstructions(DenseMap &SlotRemap) { // We've replaced IR-level uses of the remapped allocas, so we only // need to replace direct uses here. - if (!isa(V)) + const AllocaInst *AI = dyn_cast_or_null(MMO->getValue()); + if (!AI) continue; - const AllocaInst *AI= cast(V); if (!Allocas.count(AI)) continue; diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp index 78187f85df7..ea65b05d315 100644 --- a/lib/CodeGen/StackSlotColoring.cpp +++ b/lib/CodeGen/StackSlotColoring.cpp @@ -161,13 +161,12 @@ void StackSlotColoring::ScanForSpillSlotRefs(MachineFunction &MF) { for (MachineInstr::mmo_iterator MMOI = MI->memoperands_begin(), EE = MI->memoperands_end(); MMOI != EE; ++MMOI) { MachineMemOperand *MMO = *MMOI; - if (const Value *V = MMO->getValue()) { - if (const FixedStackPseudoSourceValue *FSV = - dyn_cast(V)) { - int FI = FSV->getFrameIndex(); - if (FI >= 0) - SSRefs[FI].push_back(MMO); - } + if (const FixedStackPseudoSourceValue *FSV = + dyn_cast_or_null( + MMO->getPseudoValue())) { + int FI = FSV->getFrameIndex(); + if (FI >= 0) + SSRefs[FI].push_back(MMO); } } } @@ -310,7 +309,7 @@ bool StackSlotColoring::ColorSlots(MachineFunction &MF) { if (NewFI == -1 || (NewFI == (int)SS)) continue; - const Value *NewSV = PseudoSourceValue::getFixedStack(NewFI); + const PseudoSourceValue *NewSV = PseudoSourceValue::getFixedStack(NewFI); SmallVectorImpl &RefMMOs = SSRefs[SS]; for (unsigned i = 0, e = RefMMOs.size(); i != e; ++i) RefMMOs[i]->setValue(NewSV); diff --git a/lib/CodeGen/TargetInstrInfo.cpp b/lib/CodeGen/TargetInstrInfo.cpp index 983c493f311..c4c9c83120f 100644 --- a/lib/CodeGen/TargetInstrInfo.cpp +++ b/lib/CodeGen/TargetInstrInfo.cpp @@ -250,13 +250,15 @@ bool TargetInstrInfo::hasLoadFromStackSlot(const MachineInstr *MI, oe = MI->memoperands_end(); o != oe; ++o) { - if ((*o)->isLoad() && (*o)->getValue()) + if ((*o)->isLoad()) { if (const FixedStackPseudoSourceValue *Value = - dyn_cast((*o)->getValue())) { + dyn_cast_or_null( + (*o)->getPseudoValue())) { FrameIndex = Value->getFrameIndex(); MMO = *o; return true; } + } } return false; } @@ -268,13 +270,15 @@ bool TargetInstrInfo::hasStoreToStackSlot(const MachineInstr *MI, oe = MI->memoperands_end(); o != oe; ++o) { - if ((*o)->isStore() && (*o)->getValue()) + if ((*o)->isStore()) { if (const FixedStackPseudoSourceValue *Value = - dyn_cast((*o)->getValue())) { + dyn_cast_or_null( + (*o)->getPseudoValue())) { FrameIndex = Value->getFrameIndex(); MMO = *o; return true; } + } } return false; } diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp index f30a1f36dda..81b66752154 100644 --- a/lib/IR/AsmWriter.cpp +++ b/lib/IR/AsmWriter.cpp @@ -1126,12 +1126,6 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Value *V, return; } - if (V->getValueID() == Value::PseudoSourceValueVal || - V->getValueID() == Value::FixedStackPseudoSourceValueVal) { - V->print(Out); - return; - } - char Prefix = '%'; int Slot; // If we have a SlotTracker, use it. diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp index 6d199fe9576..5ffd5a4cc0a 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1659,7 +1659,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, Flags.getByValAlign(), /*isVolatile = */ false, /*alwaysInline = */ false, - DstInfo, MachinePointerInfo(0)); + DstInfo, MachinePointerInfo()); MemOpChains.push_back(Cpy); } else { // Normal stack argument, put it where it's needed. diff --git a/lib/Target/Mips/MipsDelaySlotFiller.cpp b/lib/Target/Mips/MipsDelaySlotFiller.cpp index eef9f38daf8..a1a0e063887 100644 --- a/lib/Target/Mips/MipsDelaySlotFiller.cpp +++ b/lib/Target/Mips/MipsDelaySlotFiller.cpp @@ -142,19 +142,21 @@ namespace { MemDefsUses(const MachineFrameInfo *MFI); private: + typedef PointerUnion ValueType; + virtual bool hasHazard_(const MachineInstr &MI); /// Update Defs and Uses. Return true if there exist dependences that /// disqualify the delay slot candidate between V and values in Uses and /// Defs. - bool updateDefsUses(const Value *V, bool MayStore); + bool updateDefsUses(ValueType V, bool MayStore); /// Get the list of underlying objects of MI's memory operand. bool getUnderlyingObjects(const MachineInstr &MI, - SmallVectorImpl &Objects) const; + SmallVectorImpl &Objects) const; const MachineFrameInfo *MFI; - SmallPtrSet Uses, Defs; + SmallPtrSet Uses, Defs; /// Flags indicating whether loads or stores with no underlying objects have /// been seen. @@ -399,16 +401,15 @@ bool LoadFromStackOrConst::hasHazard_(const MachineInstr &MI) { if (MI.mayStore()) return true; - if (!MI.hasOneMemOperand() || !(*MI.memoperands_begin())->getValue()) + if (!MI.hasOneMemOperand() || !(*MI.memoperands_begin())->getPseudoValue()) return true; - const Value *V = (*MI.memoperands_begin())->getValue(); - - if (isa(V)) - return false; - - if (const PseudoSourceValue *PSV = dyn_cast(V)) - return !PSV->isConstant(0) && V != PseudoSourceValue::getStack(); + if (const PseudoSourceValue *PSV = + (*MI.memoperands_begin())->getPseudoValue()) { + if (isa(PSV)) + return false; + return !PSV->isConstant(0) && PSV != PseudoSourceValue::getStack(); + } return true; } @@ -419,11 +420,11 @@ MemDefsUses::MemDefsUses(const MachineFrameInfo *MFI_) bool MemDefsUses::hasHazard_(const MachineInstr &MI) { bool HasHazard = false; - SmallVector Objs; + SmallVector Objs; // Check underlying object list. if (getUnderlyingObjects(MI, Objs)) { - for (SmallVectorImpl::const_iterator I = Objs.begin(); + for (SmallVectorImpl::const_iterator I = Objs.begin(); I != Objs.end(); ++I) HasHazard |= updateDefsUses(*I, MI.mayStore()); @@ -440,7 +441,7 @@ bool MemDefsUses::hasHazard_(const MachineInstr &MI) { return HasHazard; } -bool MemDefsUses::updateDefsUses(const Value *V, bool MayStore) { +bool MemDefsUses::updateDefsUses(ValueType V, bool MayStore) { if (MayStore) return !Defs.insert(V) || Uses.count(V) || SeenNoObjStore || SeenNoObjLoad; @@ -450,10 +451,20 @@ bool MemDefsUses::updateDefsUses(const Value *V, bool MayStore) { bool MemDefsUses:: getUnderlyingObjects(const MachineInstr &MI, - SmallVectorImpl &Objects) const { - if (!MI.hasOneMemOperand() || !(*MI.memoperands_begin())->getValue()) + SmallVectorImpl &Objects) const { + if (!MI.hasOneMemOperand() || + (!(*MI.memoperands_begin())->getValue() && + !(*MI.memoperands_begin())->getPseudoValue())) return false; + if (const PseudoSourceValue *PSV = + (*MI.memoperands_begin())->getPseudoValue()) { + if (!PSV->isAliased(MFI)) + return false; + Objects.push_back(PSV); + return true; + } + const Value *V = (*MI.memoperands_begin())->getValue(); SmallVector Objs; @@ -461,10 +472,7 @@ getUnderlyingObjects(const MachineInstr &MI, for (SmallVectorImpl::iterator I = Objs.begin(), E = Objs.end(); I != E; ++I) { - if (const PseudoSourceValue *PSV = dyn_cast(*I)) { - if (PSV->isAliased(MFI)) - return false; - } else if (!isIdentifiedObject(V)) + if (!isIdentifiedObject(V)) return false; Objects.push_back(*I); diff --git a/lib/Target/Mips/MipsISelLowering.cpp b/lib/Target/Mips/MipsISelLowering.cpp index 48c49dbfe27..e7120edb843 100644 --- a/lib/Target/Mips/MipsISelLowering.cpp +++ b/lib/Target/Mips/MipsISelLowering.cpp @@ -3550,7 +3550,7 @@ passByValArg(SDValue Chain, SDLoc DL, DAG.getIntPtrConstant(ByVal.Address)); Chain = DAG.getMemcpy(Chain, DL, Dst, Src, DAG.getConstant(MemCpySize, PtrTy), Alignment, /*isVolatile=*/false, /*AlwaysInline=*/false, - MachinePointerInfo(0), MachinePointerInfo(0)); + MachinePointerInfo(), MachinePointerInfo()); MemOpChains.push_back(Chain); } @@ -3592,7 +3592,7 @@ void MipsTargetLowering::writeVarArgRegs(std::vector &OutChains, SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy()); SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff, MachinePointerInfo(), false, false, 0); - cast(Store.getNode())->getMemOperand()->setValue(0); + cast(Store.getNode())->getMemOperand()->setValue((Value*)0); OutChains.push_back(Store); } } diff --git a/lib/Target/Mips/MipsOptimizePICCall.cpp b/lib/Target/Mips/MipsOptimizePICCall.cpp index db270f38c71..cf60d0d74db 100644 --- a/lib/Target/Mips/MipsOptimizePICCall.cpp +++ b/lib/Target/Mips/MipsOptimizePICCall.cpp @@ -35,11 +35,13 @@ static cl::opt EraseGPOpnd("mips-erase-gp-opnd", cl::Hidden); namespace { +typedef PointerUnion ValueType; + typedef std::pair CntRegP; typedef RecyclingAllocator > + ScopedHashTableVal > AllocatorTy; -typedef ScopedHashTable, +typedef ScopedHashTable, AllocatorTy> ScopedHTType; class MBBInfo { @@ -78,18 +80,18 @@ private: /// and the underlying object in Reg and Val respectively, if the function's /// address can be resolved lazily. bool isCallViaRegister(MachineInstr &MI, unsigned &Reg, - const Value *&Val) const; + ValueType &Val) const; /// \brief Return the number of instructions that dominate the current /// instruction and load the function address from object Entry. - unsigned getCount(const Value *Entry); + unsigned getCount(ValueType Entry); /// \brief Return the destination virtual register of the last instruction /// that loads from object Entry. - unsigned getReg(const Value *Entry); + unsigned getReg(ValueType Entry); /// \brief Update ScopedHT. - void incCntAndSetReg(const Value *Entry, unsigned Reg); + void incCntAndSetReg(ValueType Entry, unsigned Reg); ScopedHTType ScopedHT; static char ID; @@ -210,7 +212,7 @@ bool OptimizePICCall::visitNode(MBBInfo &MBBI) { for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E; ++I) { unsigned Reg; - const Value *Entry; + ValueType Entry; // Skip instructions that are not call instructions via registers. if (!isCallViaRegister(*I, Reg, Entry)) @@ -242,7 +244,7 @@ bool OptimizePICCall::visitNode(MBBInfo &MBBI) { } bool OptimizePICCall::isCallViaRegister(MachineInstr &MI, unsigned &Reg, - const Value *&Val) const { + ValueType &Val) const { if (!MI.isCall()) return false; @@ -254,7 +256,7 @@ bool OptimizePICCall::isCallViaRegister(MachineInstr &MI, unsigned &Reg, // Get the instruction that loads the function address from the GOT. Reg = MO->getReg(); - Val = 0; + Val = (Value*)0; MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); MachineInstr *DefMI = MRI.getVRegDef(Reg); @@ -273,20 +275,22 @@ bool OptimizePICCall::isCallViaRegister(MachineInstr &MI, unsigned &Reg, // Return the underlying object for the GOT entry in Val. assert(DefMI->hasOneMemOperand()); Val = (*DefMI->memoperands_begin())->getValue(); + if (!Val) + Val = (*DefMI->memoperands_begin())->getPseudoValue(); return true; } -unsigned OptimizePICCall::getCount(const Value *Entry) { +unsigned OptimizePICCall::getCount(ValueType Entry) { return ScopedHT.lookup(Entry).first; } -unsigned OptimizePICCall::getReg(const Value *Entry) { +unsigned OptimizePICCall::getReg(ValueType Entry) { unsigned Reg = ScopedHT.lookup(Entry).second; assert(Reg); return Reg; } -void OptimizePICCall::incCntAndSetReg(const Value *Entry, unsigned Reg) { +void OptimizePICCall::incCntAndSetReg(ValueType Entry, unsigned Reg) { CntRegP P = ScopedHT.lookup(Entry); ScopedHT.insert(Entry, std::make_pair(P.first + 1, Reg)); } diff --git a/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp index 9fd4bb1c699..7e1dcdfa5ae 100644 --- a/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp +++ b/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp @@ -267,7 +267,7 @@ SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { static unsigned int getCodeAddrSpace(MemSDNode *N, const NVPTXSubtarget &Subtarget) { - const Value *Src = N->getSrcValue(); + const Value *Src = N->getMemOperand()->getValue(); if (!Src) return NVPTX::PTXLdStInstCode::GENERIC; @@ -3061,9 +3061,13 @@ bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, // the classof() for MemSDNode does not include MemIntrinsicSDNode // (See SelectionDAGNodes.h). So we need to check for both. if (MemSDNode *mN = dyn_cast(N)) { - Src = mN->getSrcValue(); + if (spN == 0 && mN->getMemOperand()->getPseudoValue()) + return true; + Src = mN->getMemOperand()->getValue(); } else if (MemSDNode *mN = dyn_cast(N)) { - Src = mN->getSrcValue(); + if (spN == 0 && mN->getMemOperand()->getPseudoValue()) + return true; + Src = mN->getMemOperand()->getValue(); } if (!Src) return false; diff --git a/lib/Target/PowerPC/PPCISelLowering.cpp b/lib/Target/PowerPC/PPCISelLowering.cpp index 3be0f92df42..0c0ce5ac508 100644 --- a/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/lib/Target/PowerPC/PPCISelLowering.cpp @@ -3315,8 +3315,8 @@ CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, SDLoc dl) { SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), - false, false, MachinePointerInfo(0), - MachinePointerInfo(0)); + false, false, MachinePointerInfo(), + MachinePointerInfo()); } /// LowerMemOpCallTo - Store the argument to the stack or remember it in case of diff --git a/lib/Target/R600/AMDGPUISelDAGToDAG.cpp b/lib/Target/R600/AMDGPUISelDAGToDAG.cpp index f0ff0f9e146..574bef2cca4 100644 --- a/lib/Target/R600/AMDGPUISelDAGToDAG.cpp +++ b/lib/Target/R600/AMDGPUISelDAGToDAG.cpp @@ -61,6 +61,7 @@ private: bool SelectADDR64(SDValue N, SDValue &R1, SDValue &R2); static bool checkType(const Value *ptr, unsigned int addrspace); + static bool checkPrivateAddress(const MachineMemOperand *Op); static bool isGlobalStore(const StoreSDNode *N); static bool isPrivateStore(const StoreSDNode *N); @@ -431,6 +432,7 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) { bool AMDGPUDAGToDAGISel::checkType(const Value *ptr, unsigned int addrspace) { + assert(addrspace != 0 && "Use checkPrivateAddress instead."); if (!ptr) { return false; } @@ -438,29 +440,41 @@ bool AMDGPUDAGToDAGISel::checkType(const Value *ptr, unsigned int addrspace) { return dyn_cast(ptrType)->getAddressSpace() == addrspace; } +bool AMDGPUDAGToDAGISel::checkPrivateAddress(const MachineMemOperand *Op) { + if (Op->getPseudoValue()) return true; + const Value *ptr = Op->getValue(); + if (!ptr) return false; + PointerType *ptrType = dyn_cast(ptr->getType()); + return ptrType->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS; +} + bool AMDGPUDAGToDAGISel::isGlobalStore(const StoreSDNode *N) { - return checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS); + return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS); } bool AMDGPUDAGToDAGISel::isPrivateStore(const StoreSDNode *N) { - return (!checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS) - && !checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS) - && !checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS)); + return (!checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS) + && !checkType(N->getMemOperand()->getValue(), + AMDGPUAS::GLOBAL_ADDRESS) + && !checkType(N->getMemOperand()->getValue(), + AMDGPUAS::REGION_ADDRESS)); } bool AMDGPUDAGToDAGISel::isLocalStore(const StoreSDNode *N) { - return checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS); + return checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS); } bool AMDGPUDAGToDAGISel::isRegionStore(const StoreSDNode *N) { - return checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS); + return checkType(N->getMemOperand()->getValue(), AMDGPUAS::REGION_ADDRESS); } bool AMDGPUDAGToDAGISel::isConstantLoad(const LoadSDNode *N, int CbId) const { if (CbId == -1) { - return checkType(N->getSrcValue(), AMDGPUAS::CONSTANT_ADDRESS); + return checkType(N->getMemOperand()->getValue(), + AMDGPUAS::CONSTANT_ADDRESS); } - return checkType(N->getSrcValue(), AMDGPUAS::CONSTANT_BUFFER_0 + CbId); + return checkType(N->getMemOperand()->getValue(), + AMDGPUAS::CONSTANT_BUFFER_0 + CbId); } bool AMDGPUDAGToDAGISel::isGlobalLoad(const LoadSDNode *N) const { @@ -471,27 +485,26 @@ bool AMDGPUDAGToDAGISel::isGlobalLoad(const LoadSDNode *N) const { return true; } } - return checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS); + return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS); } bool AMDGPUDAGToDAGISel::isParamLoad(const LoadSDNode *N) const { - return checkType(N->getSrcValue(), AMDGPUAS::PARAM_I_ADDRESS); + return checkType(N->getMemOperand()->getValue(), AMDGPUAS::PARAM_I_ADDRESS); } bool AMDGPUDAGToDAGISel::isLocalLoad(const LoadSDNode *N) const { - return checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS); + return checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS); } bool AMDGPUDAGToDAGISel::isRegionLoad(const LoadSDNode *N) const { - return checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS); + return checkType(N->getMemOperand()->getValue(), AMDGPUAS::REGION_ADDRESS); } bool AMDGPUDAGToDAGISel::isCPLoad(const LoadSDNode *N) const { MachineMemOperand *MMO = N->getMemOperand(); - if (checkType(N->getSrcValue(), AMDGPUAS::PRIVATE_ADDRESS)) { + if (checkPrivateAddress(N->getMemOperand())) { if (MMO) { - const Value *V = MMO->getValue(); - const PseudoSourceValue *PSV = dyn_cast(V); + const PseudoSourceValue *PSV = MMO->getPseudoValue(); if (PSV && PSV == PseudoSourceValue::getConstantPool()) { return true; } @@ -501,19 +514,19 @@ bool AMDGPUDAGToDAGISel::isCPLoad(const LoadSDNode *N) const { } bool AMDGPUDAGToDAGISel::isPrivateLoad(const LoadSDNode *N) const { - if (checkType(N->getSrcValue(), AMDGPUAS::PRIVATE_ADDRESS)) { + if (checkPrivateAddress(N->getMemOperand())) { // Check to make sure we are not a constant pool load or a constant load // that is marked as a private load if (isCPLoad(N) || isConstantLoad(N, -1)) { return false; } } - if (!checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS) - && !checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS) - && !checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS) - && !checkType(N->getSrcValue(), AMDGPUAS::CONSTANT_ADDRESS) - && !checkType(N->getSrcValue(), AMDGPUAS::PARAM_D_ADDRESS) - && !checkType(N->getSrcValue(), AMDGPUAS::PARAM_I_ADDRESS)) { + if (!checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS) + && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS) + && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::REGION_ADDRESS) + && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::CONSTANT_ADDRESS) + && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::PARAM_D_ADDRESS) + && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::PARAM_I_ADDRESS)){ return true; } return false; diff --git a/lib/Target/R600/AMDGPUISelLowering.cpp b/lib/Target/R600/AMDGPUISelLowering.cpp index 95ff0ca5b1c..523ea10a976 100644 --- a/lib/Target/R600/AMDGPUISelLowering.cpp +++ b/lib/Target/R600/AMDGPUISelLowering.cpp @@ -778,7 +778,8 @@ SDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { // Lower loads constant address space global variable loads if (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS && - isa(GetUnderlyingObject(Load->getPointerInfo().V))) { + isa( + GetUnderlyingObject(Load->getMemOperand()->getValue()))) { SDValue Ptr = DAG.getZExtOrTrunc(Load->getBasePtr(), DL, getPointerTy(AMDGPUAS::PRIVATE_ADDRESS)); diff --git a/lib/Target/R600/R600ISelLowering.cpp b/lib/Target/R600/R600ISelLowering.cpp index be810851a17..489e63aa45b 100644 --- a/lib/Target/R600/R600ISelLowering.cpp +++ b/lib/Target/R600/R600ISelLowering.cpp @@ -1233,8 +1233,8 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) || (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) { SDValue Result; - if (isa(LoadNode->getSrcValue()) || - isa(LoadNode->getSrcValue()) || + if (isa(LoadNode->getMemOperand()->getValue()) || + isa(LoadNode->getMemOperand()->getValue()) || isa(Ptr)) { SDValue Slots[4]; for (unsigned i = 0; i < 4; i++) { diff --git a/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp index f46eb16be77..81d744f7d13 100644 --- a/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ b/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -986,8 +986,8 @@ bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store, return true; // Otherwise we need to check whether there's an alias. - const Value *V1 = Load->getSrcValue(); - const Value *V2 = Store->getSrcValue(); + const Value *V1 = Load->getMemOperand()->getValue(); + const Value *V2 = Store->getMemOperand()->getValue(); if (!V1 || !V2) return false; diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index c2184b548c2..e78394e121c 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -13726,8 +13726,7 @@ static SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) { cast(Node)->getMemoryVT(), Node->getOperand(0), Node->getOperand(1), negOp, - cast(Node)->getSrcValue(), - cast(Node)->getAlignment(), + cast(Node)->getMemOperand(), cast(Node)->getOrdering(), cast(Node)->getSynchScope()); } -- 2.34.1