X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=blobdiff_plain;f=include%2Fllvm%2FAnalysis%2FLoopAccessAnalysis.h;h=38f09a63e6477586ab542ba1361da7fed653a87d;hp=323af980ddc7c551d80fbf9cd1ea845419faaee3;hb=5ba219fe8d140cec2a90aa99e18592c2143e158e;hpb=5a51d864b9041151152a555fb2051357e031659c diff --git a/include/llvm/Analysis/LoopAccessAnalysis.h b/include/llvm/Analysis/LoopAccessAnalysis.h index 323af980ddc..38f09a63e64 100644 --- a/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/include/llvm/Analysis/LoopAccessAnalysis.h @@ -86,80 +86,406 @@ struct VectorizerParams { static unsigned RuntimeMemoryCheckThreshold; }; -/// \brief Drive the analysis of memory accesses in the loop +/// \brief Checks memory dependences among accesses to the same underlying +/// object to determine whether there vectorization is legal or not (and at +/// which vectorization factor). /// -/// This class is responsible for analyzing the memory accesses of a loop. It -/// collects the accesses and then its main helper the AccessAnalysis class -/// finds and categorizes the dependences in buildDependenceSets. +/// Note: This class will compute a conservative dependence for access to +/// different underlying pointers. Clients, such as the loop vectorizer, will +/// sometimes deal these potential dependencies by emitting runtime checks. /// -/// For memory dependences that can be analyzed at compile time, it determines -/// whether the dependence is part of cycle inhibiting vectorization. This work -/// is delegated to the MemoryDepChecker class. +/// We use the ScalarEvolution framework to symbolically evalutate access +/// functions pairs. Since we currently don't restructure the loop we can rely +/// on the program order of memory accesses to determine their safety. +/// At the moment we will only deem accesses as safe for: +/// * A negative constant distance assuming program order. /// -/// For memory dependences that cannot be determined at compile time, it -/// generates run-time checks to prove independence. This is done by -/// AccessAnalysis::canCheckPtrAtRT and the checks are maintained by the -/// RuntimePointerCheck class. -class LoopAccessInfo { +/// Safe: tmp = a[i + 1]; OR a[i + 1] = x; +/// a[i] = tmp; y = a[i]; +/// +/// The latter case is safe because later checks guarantuee that there can't +/// be a cycle through a phi node (that is, we check that "x" and "y" is not +/// the same variable: a header phi can only be an induction or a reduction, a +/// reduction can't have a memory sink, an induction can't have a memory +/// source). This is important and must not be violated (or we have to +/// resort to checking for cycles through memory). +/// +/// * A positive constant distance assuming program order that is bigger +/// than the biggest memory access. +/// +/// tmp = a[i] OR b[i] = x +/// a[i+2] = tmp y = b[i+2]; +/// +/// Safe distance: 2 x sizeof(a[0]), and 2 x sizeof(b[0]), respectively. +/// +/// * Zero distances and all accesses have the same size. +/// +class MemoryDepChecker { public: - /// This struct holds information about the memory runtime legality check that - /// a group of pointers do not overlap. - struct RuntimePointerCheck { - RuntimePointerCheck() : Need(false) {} - - /// Reset the state of the pointer runtime information. - void reset() { - Need = false; - Pointers.clear(); - Starts.clear(); - Ends.clear(); - IsWritePtr.clear(); - DependencySetId.clear(); - AliasSetId.clear(); - } + typedef PointerIntPair MemAccessInfo; + typedef SmallPtrSet MemAccessInfoSet; + /// \brief Set of potential dependent memory accesses. + typedef EquivalenceClasses DepCandidates; + + /// \brief Dependece between memory access instructions. + struct Dependence { + /// \brief The type of the dependence. + enum DepType { + // No dependence. + NoDep, + // We couldn't determine the direction or the distance. + Unknown, + // Lexically forward. + Forward, + // Forward, but if vectorized, is likely to prevent store-to-load + // forwarding. + ForwardButPreventsForwarding, + // Lexically backward. + Backward, + // Backward, but the distance allows a vectorization factor of + // MaxSafeDepDistBytes. + BackwardVectorizable, + // Same, but may prevent store-to-load forwarding. + BackwardVectorizableButPreventsForwarding + }; + + /// \brief String version of the types. + static const char *DepName[]; + + /// \brief Index of the source of the dependence in the InstMap vector. + unsigned Source; + /// \brief Index of the destination of the dependence in the InstMap vector. + unsigned Destination; + /// \brief The type of the dependence. + DepType Type; + + Dependence(unsigned Source, unsigned Destination, DepType Type) + : Source(Source), Destination(Destination), Type(Type) {} + + /// \brief Dependence types that don't prevent vectorization. + static bool isSafeForVectorization(DepType Type); + + /// \brief Dependence types that can be queried from the analysis. + static bool isInterestingDependence(DepType Type); + + /// \brief Lexically backward dependence types. + bool isPossiblyBackward() const; + + /// \brief Print the dependence. \p Instr is used to map the instruction + /// indices to instructions. + void print(raw_ostream &OS, unsigned Depth, + const SmallVectorImpl &Instrs) const; + }; + + MemoryDepChecker(ScalarEvolution *Se, const Loop *L) + : SE(Se), InnermostLoop(L), AccessIdx(0), + ShouldRetryWithRuntimeCheck(false), SafeForVectorization(true), + RecordInterestingDependences(true) {} + + /// \brief Register the location (instructions are given increasing numbers) + /// of a write access. + void addAccess(StoreInst *SI) { + Value *Ptr = SI->getPointerOperand(); + Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx); + InstMap.push_back(SI); + ++AccessIdx; + } + + /// \brief Register the location (instructions are given increasing numbers) + /// of a write access. + void addAccess(LoadInst *LI) { + Value *Ptr = LI->getPointerOperand(); + Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx); + InstMap.push_back(LI); + ++AccessIdx; + } + + /// \brief Check whether the dependencies between the accesses are safe. + /// + /// Only checks sets with elements in \p CheckDeps. + bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoSet &CheckDeps, + const ValueToValueMap &Strides); + + /// \brief No memory dependence was encountered that would inhibit + /// vectorization. + bool isSafeForVectorization() const { return SafeForVectorization; } + + /// \brief The maximum number of bytes of a vector register we can vectorize + /// the accesses safely with. + unsigned getMaxSafeDepDistBytes() { return MaxSafeDepDistBytes; } + + /// \brief In same cases when the dependency check fails we can still + /// vectorize the loop with a dynamic array access check. + bool shouldRetryWithRuntimeCheck() { return ShouldRetryWithRuntimeCheck; } + + /// \brief Returns the interesting dependences. If null is returned we + /// exceeded the MaxInterestingDependence threshold and this information is + /// not available. + const SmallVectorImpl *getInterestingDependences() const { + return RecordInterestingDependences ? &InterestingDependences : nullptr; + } - /// Insert a pointer and calculate the start and end SCEVs. - void insert(ScalarEvolution *SE, Loop *Lp, Value *Ptr, bool WritePtr, - unsigned DepSetId, unsigned ASId, - const ValueToValueMap &Strides); + void clearInterestingDependences() { InterestingDependences.clear(); } + + /// \brief The vector of memory access instructions. The indices are used as + /// instruction identifiers in the Dependence class. + const SmallVectorImpl &getMemoryInstructions() const { + return InstMap; + } + + /// \brief Find the set of instructions that read or write via \p Ptr. + SmallVector getInstructionsForAccess(Value *Ptr, + bool isWrite) const; + +private: + ScalarEvolution *SE; + const Loop *InnermostLoop; + + /// \brief Maps access locations (ptr, read/write) to program order. + DenseMap > Accesses; + + /// \brief Memory access instructions in program order. + SmallVector InstMap; + + /// \brief The program order index to be used for the next instruction. + unsigned AccessIdx; + + // We can access this many bytes in parallel safely. + unsigned MaxSafeDepDistBytes; - /// \brief No run-time memory checking is necessary. - bool empty() const { return Pointers.empty(); } + /// \brief If we see a non-constant dependence distance we can still try to + /// vectorize this loop with runtime checks. + bool ShouldRetryWithRuntimeCheck; - /// \brief Decide whether we need to issue a run-time check for pointer at - /// index \p I and \p J to prove their independence. - bool needsChecking(unsigned I, unsigned J) const; + /// \brief No memory dependence was encountered that would inhibit + /// vectorization. + bool SafeForVectorization; - /// \brief Print the list run-time memory checks necessary. - void print(raw_ostream &OS, unsigned Depth = 0) const; + //// \brief True if InterestingDependences reflects the dependences in the + //// loop. If false we exceeded MaxInterestingDependence and + //// InterestingDependences is invalid. + bool RecordInterestingDependences; - /// This flag indicates if we need to add the runtime check. - bool Need; - /// Holds the pointers that we need to check. - SmallVector, 2> Pointers; + /// \brief Interesting memory dependences collected during the analysis as + /// defined by isInterestingDependence. Only valid if + /// RecordInterestingDependences is true. + SmallVector InterestingDependences; + + /// \brief Check whether there is a plausible dependence between the two + /// accesses. + /// + /// Access \p A must happen before \p B in program order. The two indices + /// identify the index into the program order map. + /// + /// This function checks whether there is a plausible dependence (or the + /// absence of such can't be proved) between the two accesses. If there is a + /// plausible dependence but the dependence distance is bigger than one + /// element access it records this distance in \p MaxSafeDepDistBytes (if this + /// distance is smaller than any other distance encountered so far). + /// Otherwise, this function returns true signaling a possible dependence. + Dependence::DepType isDependent(const MemAccessInfo &A, unsigned AIdx, + const MemAccessInfo &B, unsigned BIdx, + const ValueToValueMap &Strides); + + /// \brief Check whether the data dependence could prevent store-load + /// forwarding. + bool couldPreventStoreLoadForward(unsigned Distance, unsigned TypeByteSize); +}; + +/// \brief Holds information about the memory runtime legality checks to verify +/// that a group of pointers do not overlap. +class RuntimePointerChecking { +public: + struct PointerInfo { + /// Holds the pointer value that we need to check. + TrackingVH PointerValue; /// Holds the pointer value at the beginning of the loop. - SmallVector Starts; + const SCEV *Start; /// Holds the pointer value at the end of the loop. - SmallVector Ends; + const SCEV *End; /// Holds the information if this pointer is used for writing to memory. - SmallVector IsWritePtr; + bool IsWritePtr; /// Holds the id of the set of pointers that could be dependent because of a /// shared underlying object. - SmallVector DependencySetId; + unsigned DependencySetId; /// Holds the id of the disjoint alias set to which this pointer belongs. - SmallVector AliasSetId; + unsigned AliasSetId; + /// SCEV for the access. + const SCEV *Expr; + + PointerInfo(Value *PointerValue, const SCEV *Start, const SCEV *End, + bool IsWritePtr, unsigned DependencySetId, unsigned AliasSetId, + const SCEV *Expr) + : PointerValue(PointerValue), Start(Start), End(End), + IsWritePtr(IsWritePtr), DependencySetId(DependencySetId), + AliasSetId(AliasSetId), Expr(Expr) {} + }; + + RuntimePointerChecking(ScalarEvolution *SE) : Need(false), SE(SE) {} + + /// Reset the state of the pointer runtime information. + void reset() { + Need = false; + Pointers.clear(); + Checks.clear(); + } + + /// Insert a pointer and calculate the start and end SCEVs. + void insert(Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId, + unsigned ASId, const ValueToValueMap &Strides); + + /// \brief No run-time memory checking is necessary. + bool empty() const { return Pointers.empty(); } + + /// A grouping of pointers. A single memcheck is required between + /// two groups. + struct CheckingPtrGroup { + /// \brief Create a new pointer checking group containing a single + /// pointer, with index \p Index in RtCheck. + CheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck) + : RtCheck(RtCheck), High(RtCheck.Pointers[Index].End), + Low(RtCheck.Pointers[Index].Start) { + Members.push_back(Index); + } + + /// \brief Tries to add the pointer recorded in RtCheck at index + /// \p Index to this pointer checking group. We can only add a pointer + /// to a checking group if we will still be able to get + /// the upper and lower bounds of the check. Returns true in case + /// of success, false otherwise. + bool addPointer(unsigned Index); + + /// Constitutes the context of this pointer checking group. For each + /// pointer that is a member of this group we will retain the index + /// at which it appears in RtCheck. + RuntimePointerChecking &RtCheck; + /// The SCEV expression which represents the upper bound of all the + /// pointers in this group. + const SCEV *High; + /// The SCEV expression which represents the lower bound of all the + /// pointers in this group. + const SCEV *Low; + /// Indices of all the pointers that constitute this grouping. + SmallVector Members; }; - LoopAccessInfo(Loop *L, ScalarEvolution *SE, const DataLayout *DL, + /// \brief A memcheck which made up of a pair of grouped pointers. + /// + /// These *have* to be const for now, since checks are generated from + /// CheckingPtrGroups in LAI::addRuntimeCheck which is a const member + /// function. FIXME: once check-generation is moved inside this class (after + /// the PtrPartition hack is removed), we could drop const. + typedef std::pair + PointerCheck; + + /// \brief Generate the checks and store it. This also performs the grouping + /// of pointers to reduce the number of memchecks necessary. + void generateChecks(MemoryDepChecker::DepCandidates &DepCands, + bool UseDependencies); + + /// \brief Returns the checks that generateChecks created. + const SmallVectorImpl &getChecks() const { return Checks; } + + /// \brief Decide if we need to add a check between two groups of pointers, + /// according to needsChecking. + bool needsChecking(const CheckingPtrGroup &M, const CheckingPtrGroup &N, + const SmallVectorImpl *PtrPartition) const; + + /// \brief Returns the number of run-time checks required according to + /// needsChecking. + unsigned getNumberOfChecks() const { return Checks.size(); } + + /// \brief Print the list run-time memory checks necessary. + void print(raw_ostream &OS, unsigned Depth = 0) const; + + /// Print \p Checks. + void printChecks(raw_ostream &OS, const SmallVectorImpl &Checks, + unsigned Depth = 0) const; + + /// This flag indicates if we need to add the runtime check. + bool Need; + + /// Information about the pointers that may require checking. + SmallVector Pointers; + + /// Holds a partitioning of pointers into "check groups". + SmallVector CheckingGroups; + + /// \brief Check if pointers are in the same partition + /// + /// \p PtrToPartition contains the partition number for pointers (-1 if the + /// pointer belongs to multiple partitions). + static bool + arePointersInSamePartition(const SmallVectorImpl &PtrToPartition, + unsigned PtrIdx1, unsigned PtrIdx2); + + /// \brief Decide whether we need to issue a run-time check for pointer at + /// index \p I and \p J to prove their independence. + /// + /// If \p PtrPartition is set, it contains the partition number for + /// pointers (-1 if the pointer belongs to multiple partitions). In this + /// case omit checks between pointers belonging to the same partition. + bool needsChecking(unsigned I, unsigned J, + const SmallVectorImpl *PtrPartition = nullptr) const; + +private: + /// \brief Groups pointers such that a single memcheck is required + /// between two different groups. This will clear the CheckingGroups vector + /// and re-compute it. We will only group dependecies if \p UseDependencies + /// is true, otherwise we will create a separate group for each pointer. + void groupChecks(MemoryDepChecker::DepCandidates &DepCands, + bool UseDependencies); + + /// Generate the checks and return them. + /// + /// \p PtrToPartition contains the partition number for pointers. If passed, + /// omit checks between pointers belonging to the same partition. Partition + /// number -1 means that the pointer is used in multiple partitions. In this + /// case we can't safely omit the check. + SmallVector + generateChecks(const SmallVectorImpl *PtrPartition = nullptr) const; + + /// Holds a pointer to the ScalarEvolution analysis. + ScalarEvolution *SE; + + /// \brief Set of run-time checks required to establish independence of + /// otherwise may-aliasing pointers in the loop. + SmallVector Checks; +}; + +/// \brief Drive the analysis of memory accesses in the loop +/// +/// This class is responsible for analyzing the memory accesses of a loop. It +/// collects the accesses and then its main helper the AccessAnalysis class +/// finds and categorizes the dependences in buildDependenceSets. +/// +/// For memory dependences that can be analyzed at compile time, it determines +/// whether the dependence is part of cycle inhibiting vectorization. This work +/// is delegated to the MemoryDepChecker class. +/// +/// For memory dependences that cannot be determined at compile time, it +/// generates run-time checks to prove independence. This is done by +/// AccessAnalysis::canCheckPtrAtRT and the checks are maintained by the +/// RuntimePointerCheck class. +class LoopAccessInfo { +public: + LoopAccessInfo(Loop *L, ScalarEvolution *SE, const DataLayout &DL, const TargetLibraryInfo *TLI, AliasAnalysis *AA, - DominatorTree *DT, const ValueToValueMap &Strides); + DominatorTree *DT, LoopInfo *LI, + const ValueToValueMap &Strides); /// Return true we can analyze the memory accesses in the loop and there are /// no memory dependence cycles. bool canVectorizeMemory() const { return CanVecMem; } - const RuntimePointerCheck *getRuntimePointerCheck() const { - return &PtrRtCheck; + const RuntimePointerChecking *getRuntimePointerChecking() const { + return &PtrRtChecking; + } + + /// \brief Number of memchecks required to prove independence of otherwise + /// may-alias pointers. + unsigned getNumRuntimePointerChecks() const { + return PtrRtChecking.getNumberOfChecks(); } /// Return true if the block BB needs to be predicated in order for the loop @@ -180,12 +506,33 @@ public: /// instruction generated in possibly a sequence of instructions and the /// second value is the final comparator value or NULL if no check is needed. std::pair - addRuntimeCheck(Instruction *Loc) const; + addRuntimeCheck(Instruction *Loc) const; + + /// \brief Generete the instructions for the checks in \p PointerChecks. + /// + /// Returns a pair of instructions where the first element is the first + /// instruction generated in possibly a sequence of instructions and the + /// second value is the final comparator value or NULL if no check is needed. + std::pair + addRuntimeCheck(Instruction *Loc, + const SmallVectorImpl + &PointerChecks) const; /// \brief The diagnostics report generated for the analysis. E.g. why we /// couldn't analyze the loop. const Optional &getReport() const { return Report; } + /// \brief the Memory Dependence Checker which can determine the + /// loop-independent and loop-carried dependences between memory accesses. + const MemoryDepChecker &getDepChecker() const { return DepChecker; } + + /// \brief Return the list of instructions that use \p Ptr to read or write + /// memory. + SmallVector getInstructionsForAccess(Value *Ptr, + bool isWrite) const { + return DepChecker.getInstructionsForAccess(Ptr, isWrite); + } + /// \brief Print the information about the memory accesses in the loop. void print(raw_ostream &OS, unsigned Depth = 0) const; @@ -194,6 +541,13 @@ public: /// Only used in DEBUG build but we don't want NDEBUG-dependent ABI. unsigned NumSymbolicStrides; + /// \brief Checks existence of store to invariant address inside loop. + /// If the loop has any store to invariant address, then it returns true, + /// else returns false. + bool hasStoreToLoopInvariantAddress() const { + return StoreToLoopInvariantAddress; + } + private: /// \brief Analyze the loop. Substitute symbolic strides using Strides. void analyzeLoop(const ValueToValueMap &Strides); @@ -206,13 +560,19 @@ private: /// We need to check that all of the pointers in this list are disjoint /// at runtime. - RuntimePointerCheck PtrRtCheck; + RuntimePointerChecking PtrRtChecking; + + /// \brief the Memory Dependence Checker which can determine the + /// loop-independent and loop-carried dependences between memory accesses. + MemoryDepChecker DepChecker; + Loop *TheLoop; ScalarEvolution *SE; - const DataLayout *DL; + const DataLayout &DL; const TargetLibraryInfo *TLI; AliasAnalysis *AA; DominatorTree *DT; + LoopInfo *LI; unsigned NumLoads; unsigned NumStores; @@ -222,6 +582,10 @@ private: /// \brief Cache the result of analyzeLoop. bool CanVecMem; + /// \brief Indicator for storing to uniform addresses. + /// If a loop has write to a loop invariant address then it should be true. + bool StoreToLoopInvariantAddress; + /// \brief The diagnostics report generated for the analysis. E.g. why we /// couldn't analyze the loop. Optional Report; @@ -239,6 +603,11 @@ const SCEV *replaceSymbolicStrideSCEV(ScalarEvolution *SE, const ValueToValueMap &PtrToStride, Value *Ptr, Value *OrigPtr = nullptr); +/// \brief Check the stride of the pointer and ensure that it does not wrap in +/// the address space. +int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp, + const ValueToValueMap &StridesMap); + /// \brief This analysis provides dependence information for the memory accesses /// of a loop. /// @@ -280,10 +649,10 @@ private: // The used analysis passes. ScalarEvolution *SE; - const DataLayout *DL; const TargetLibraryInfo *TLI; AliasAnalysis *AA; DominatorTree *DT; + LoopInfo *LI; }; } // End llvm namespace