X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=blobdiff_plain;f=lib%2FTransforms%2FInstCombine%2FInstCombineLoadStoreAlloca.cpp;h=3813d8dd0cc0dd1bde75eb935e5943cb2dca9005;hp=b690ef5dbc43c7fa7f20e7abebd8cf5da02d6b2a;hb=529919ff310cbfce1ba55ea252ff738d5b56b93d;hpb=e78a87b633d0f66949390e45594201da7c8bc6eb diff --git a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp index b690ef5dbc4..3813d8dd0cc 100644 --- a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -11,18 +11,16 @@ // //===----------------------------------------------------------------------===// -#include "InstCombine.h" -#include "llvm/ADT/Optional.h" +#include "InstCombineInternal.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/Loads.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/PatternMatch.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" using namespace llvm; -using namespace llvm::PatternMatch; #define DEBUG_TYPE "instcombine" @@ -169,14 +167,11 @@ isOnlyCopiedFromConstantGlobal(AllocaInst *AI, Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) { // Ensure that the alloca array size argument has type intptr_t, so that // any casting is exposed early. - if (DL) { - Type *IntPtrTy = DL->getIntPtrType(AI.getType()); - if (AI.getArraySize()->getType() != IntPtrTy) { - Value *V = Builder->CreateIntCast(AI.getArraySize(), - IntPtrTy, false); - AI.setOperand(0, V); - return &AI; - } + Type *IntPtrTy = DL.getIntPtrType(AI.getType()); + if (AI.getArraySize()->getType() != IntPtrTy) { + Value *V = Builder->CreateIntCast(AI.getArraySize(), IntPtrTy, false); + AI.setOperand(0, V); + return &AI; } // Convert: alloca Ty, C - where C is a constant != 1 into: alloca [C x Ty], 1 @@ -196,9 +191,7 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) { // Now that I is pointing to the first non-allocation-inst in the block, // insert our getelementptr instruction... // - Type *IdxTy = DL - ? DL->getIntPtrType(AI.getType()) - : Type::getInt64Ty(AI.getContext()); + Type *IdxTy = DL.getIntPtrType(AI.getType()); Value *NullIdx = Constant::getNullValue(IdxTy); Value *Idx[2] = { NullIdx, NullIdx }; Instruction *GEP = @@ -213,15 +206,15 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) { } } - if (DL && AI.getAllocatedType()->isSized()) { + if (AI.getAllocatedType()->isSized()) { // If the alignment is 0 (unspecified), assign it the preferred alignment. if (AI.getAlignment() == 0) - AI.setAlignment(DL->getPrefTypeAlignment(AI.getAllocatedType())); + AI.setAlignment(DL.getPrefTypeAlignment(AI.getAllocatedType())); // Move all alloca's of zero byte objects to the entry block and merge them // together. Note that we only do this for alloca's, because malloc should // allocate and return a unique pointer, even for a zero byte allocation. - if (DL->getTypeAllocSize(AI.getAllocatedType()) == 0) { + if (DL.getTypeAllocSize(AI.getAllocatedType()) == 0) { // For a zero sized alloca there is no point in doing an array allocation. // This is helpful if the array size is a complicated expression not used // elsewhere. @@ -239,7 +232,7 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) { // dominance as the array size was forced to a constant earlier already. AllocaInst *EntryAI = dyn_cast(FirstInst); if (!EntryAI || !EntryAI->getAllocatedType()->isSized() || - DL->getTypeAllocSize(EntryAI->getAllocatedType()) != 0) { + DL.getTypeAllocSize(EntryAI->getAllocatedType()) != 0) { AI.moveBefore(FirstInst); return &AI; } @@ -248,7 +241,7 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) { // assign it the preferred alignment. if (EntryAI->getAlignment() == 0) EntryAI->setAlignment( - DL->getPrefTypeAlignment(EntryAI->getAllocatedType())); + DL.getPrefTypeAlignment(EntryAI->getAllocatedType())); // Replace this zero-sized alloca with the one at the start of the entry // block after ensuring that the address will be aligned enough for both // types. @@ -271,9 +264,8 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) { // is only subsequently read. SmallVector ToDelete; if (MemTransferInst *Copy = isOnlyCopiedFromConstantGlobal(&AI, ToDelete)) { - unsigned SourceAlign = getOrEnforceKnownAlignment(Copy->getSource(), - AI.getAlignment(), - DL, AT, &AI, DT); + unsigned SourceAlign = getOrEnforceKnownAlignment( + Copy->getSource(), AI.getAlignment(), DL, &AI, AC, DT); if (AI.getAlignment() <= SourceAlign) { DEBUG(dbgs() << "Found alloca equal to global: " << AI << '\n'); DEBUG(dbgs() << " memcpy = " << *Copy << '\n'); @@ -313,6 +305,7 @@ static LoadInst *combineLoadToNewType(InstCombiner &IC, LoadInst &LI, Type *NewT LoadInst *NewLoad = IC.Builder->CreateAlignedLoad( IC.Builder->CreateBitCast(Ptr, NewTy->getPointerTo(AS)), LI.getAlignment(), LI.getName()); + MDBuilder MDB(NewLoad->getContext()); for (const auto &MDPair : MD) { unsigned ID = MDPair.first; MDNode *N = MDPair.second; @@ -334,154 +327,84 @@ static LoadInst *combineLoadToNewType(InstCombiner &IC, LoadInst &LI, Type *NewT case LLVMContext::MD_noalias: case LLVMContext::MD_nontemporal: case LLVMContext::MD_mem_parallel_loop_access: - case LLVMContext::MD_nonnull: // All of these directly apply. NewLoad->setMetadata(ID, N); break; + case LLVMContext::MD_nonnull: + // This only directly applies if the new type is also a pointer. + if (NewTy->isPointerTy()) { + NewLoad->setMetadata(ID, N); + break; + } + // If it's integral now, translate it to !range metadata. + if (NewTy->isIntegerTy()) { + auto *ITy = cast(NewTy); + auto *NullInt = ConstantExpr::getPtrToInt( + ConstantPointerNull::get(cast(Ptr->getType())), ITy); + auto *NonNullInt = + ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1)); + NewLoad->setMetadata(LLVMContext::MD_range, + MDB.createRange(NonNullInt, NullInt)); + } + break; + case LLVMContext::MD_range: // FIXME: It would be nice to propagate this in some way, but the type - // conversions make it hard. + // conversions make it hard. If the new type is a pointer, we could + // translate it to !nonnull metadata. break; } } return NewLoad; } -/// \brief Combine integer loads to vector stores when the integers bits are -/// just a concatenation of non-integer (and non-vector) types. +/// \brief Combine a store to a new type. /// -/// This specifically matches the pattern of loading an integer, right-shifting, -/// trucating, and casting it to a non-integer type. When the shift is an exact -/// multiple of the result non-integer type's size, this is more naturally -/// expressed as a load of a vector and an extractelement. This shows up largely -/// because large integers are sometimes used to represent a "generic" load or -/// store, and only later optimization may uncover that there is a more natural -/// type to represent the load with. -static Instruction *combineIntegerLoadToVectorLoad(InstCombiner &IC, - LoadInst &LI) { - // FIXME: This is probably a reasonable transform to make for atomic stores. - assert(LI.isSimple() && "Do not call for non-simple stores!"); - - const DataLayout &DL = *IC.getDataLayout(); - unsigned BaseBits = LI.getType()->getIntegerBitWidth(); - Type *ElementTy = nullptr; - int ElementSize; - - // We match any number of element extractions from the loaded integer. Each of - // these should be RAUW'ed with an actual extract element instruction at the - // given index of a loaded vector. - struct ExtractedElement { - Instruction *Element; - int Index; - }; - SmallVector Elements; - - // Lambda to match the bit cast in the extracted element (which is the root - // pattern matched). Accepts the instruction and shifted bits, returns false - // if at any point we failed to match a suitable bitcast for element - // extraction. - auto MatchCast = [&](Instruction *I, unsigned ShiftBits) { - // The truncate must be casted to some element type. This cast can only be - // a bitcast or an inttoptr cast which is the same size. - if (!isa(I)) { - if (auto *PC = dyn_cast(I)) { - // Ensure that the pointer and integer have the exact same size. - if (PC->getOperand(0)->getType()->getIntegerBitWidth() != - DL.getTypeSizeInBits(PC->getType())) - return false; - } else { - // We only support bitcast and inttoptr. - return false; - } - } - - // All of the elements inserted need to be the same type. Either capture the - // first element type or check this element type against the previous - // element types. - if (!ElementTy) { - ElementTy = I->getType(); - // We don't handle integers, sub-vectors, or any aggregate types. We - // handle pointers and floating ponit types. - if (!ElementTy->isSingleValueType() || ElementTy->isIntegerTy() || - ElementTy->isVectorTy()) - return false; - - ElementSize = DL.getTypeSizeInBits(ElementTy); - // The base integer size and the shift need to be multiples of the element - // size in bits. - if (BaseBits % ElementSize || ShiftBits % ElementSize) - return false; - } else if (ElementTy != I->getType()) { - return false; - } - - // Compute the vector index and store the element with it. - int Index = - (DL.isLittleEndian() ? ShiftBits : BaseBits - ElementSize - ShiftBits) / - ElementSize; - ExtractedElement E = {I, Index}; - Elements.push_back(std::move(E)); - return true; - }; - - // Lambda to match the truncate in the extracted element. Accepts the - // instruction and shifted bits. Returns false if at any point we failed to - // match a suitable truncate for element extraction. - auto MatchTruncate = [&](Instruction *I, unsigned ShiftBits) { - // Handle the truncate to the bit size of the element. - auto *T = dyn_cast(I); - if (!T) - return false; - - // Walk all the users of the truncate, whuch must all be bitcasts. - for (User *TU : T->users()) - if (!MatchCast(cast(TU), ShiftBits)) - return false; - return true; - }; +/// Returns the newly created store instruction. +static StoreInst *combineStoreToNewValue(InstCombiner &IC, StoreInst &SI, Value *V) { + Value *Ptr = SI.getPointerOperand(); + unsigned AS = SI.getPointerAddressSpace(); + SmallVector, 8> MD; + SI.getAllMetadata(MD); - for (User *U : LI.users()) { - Instruction *I = cast(U); + StoreInst *NewStore = IC.Builder->CreateAlignedStore( + V, IC.Builder->CreateBitCast(Ptr, V->getType()->getPointerTo(AS)), + SI.getAlignment()); + for (const auto &MDPair : MD) { + unsigned ID = MDPair.first; + MDNode *N = MDPair.second; + // Note, essentially every kind of metadata should be preserved here! This + // routine is supposed to clone a store instruction changing *only its + // type*. The only metadata it makes sense to drop is metadata which is + // invalidated when the pointer type changes. This should essentially + // never be the case in LLVM, but we explicitly switch over only known + // metadata to be conservatively correct. If you are adding metadata to + // LLVM which pertains to stores, you almost certainly want to add it + // here. + switch (ID) { + case LLVMContext::MD_dbg: + case LLVMContext::MD_tbaa: + case LLVMContext::MD_prof: + case LLVMContext::MD_fpmath: + case LLVMContext::MD_tbaa_struct: + case LLVMContext::MD_alias_scope: + case LLVMContext::MD_noalias: + case LLVMContext::MD_nontemporal: + case LLVMContext::MD_mem_parallel_loop_access: + // All of these directly apply. + NewStore->setMetadata(ID, N); + break; - // Strip off a logical shift right and retain the shifted amount. - ConstantInt *ShiftC; - if (!match(I, m_LShr(m_Value(), m_ConstantInt(ShiftC)))) { - // This must be a direct truncate. - if (!MatchTruncate(I, 0)) - return nullptr; - continue; + case LLVMContext::MD_invariant_load: + case LLVMContext::MD_nonnull: + case LLVMContext::MD_range: + // These don't apply for stores. + break; } - - unsigned ShiftBits = ShiftC->getLimitedValue(BaseBits); - // We can't handle shifts of more than the number of bits in the integer. - if (ShiftBits == BaseBits) - return nullptr; - - // Match all the element extraction users of the shift. - for (User *IU : I->users()) - if (!MatchTruncate(cast(IU), ShiftBits)) - return nullptr; - } - - // If didn't find any extracted elements, there is nothing to do here. - if (Elements.empty()) - return nullptr; - - // Create a vector load and rewrite all of the elements extracted as - // extractelement instructions. - VectorType *VTy = VectorType::get(ElementTy, BaseBits / ElementSize); - LoadInst *NewLI = combineLoadToNewType(IC, LI, VTy); - - for (const auto &E : Elements) { - IC.Builder->SetInsertPoint(E.Element); - E.Element->replaceAllUsesWith( - IC.Builder->CreateExtractElement(NewLI, IC.Builder->getInt32(E.Index))); - IC.EraseInstFromFunction(*E.Element); } - // Return the original load to indicate it has been combined away. - return &LI; + return NewStore; } /// \brief Combine loads to match the type of value their uses after looking @@ -510,10 +433,37 @@ static Instruction *combineLoadToOperationType(InstCombiner &IC, LoadInst &LI) { if (LI.use_empty()) return nullptr; + Type *Ty = LI.getType(); + const DataLayout &DL = IC.getDataLayout(); + + // Try to canonicalize loads which are only ever stored to operate over + // integers instead of any other type. We only do this when the loaded type + // is sized and has a size exactly the same as its store size and the store + // size is a legal integer type. + if (!Ty->isIntegerTy() && Ty->isSized() && + DL.isLegalInteger(DL.getTypeStoreSizeInBits(Ty)) && + DL.getTypeStoreSizeInBits(Ty) == DL.getTypeSizeInBits(Ty)) { + if (std::all_of(LI.user_begin(), LI.user_end(), [&LI](User *U) { + auto *SI = dyn_cast(U); + return SI && SI->getPointerOperand() != &LI; + })) { + LoadInst *NewLoad = combineLoadToNewType( + IC, LI, + Type::getIntNTy(LI.getContext(), DL.getTypeStoreSizeInBits(Ty))); + // Replace all the stores with stores of the newly loaded value. + for (auto UI = LI.user_begin(), UE = LI.user_end(); UI != UE;) { + auto *SI = cast(*UI++); + IC.Builder->SetInsertPoint(SI); + combineStoreToNewValue(IC, *SI, NewLoad); + IC.EraseInstFromFunction(*SI); + } + assert(LI.use_empty() && "Failed to remove all users of the load!"); + // Return the old load so the combiner can delete it safely. + return &LI; + } + } // Fold away bit casts of the loaded value by loading the desired type. - // FIXME: We should also canonicalize loads of vectors when their elements are - // cast to other types. if (LI.hasOneUse()) if (auto *BC = dyn_cast(LI.user_back())) { LoadInst *NewLoad = combineLoadToNewType(IC, LI, BC->getDestTy()); @@ -522,11 +472,182 @@ static Instruction *combineLoadToOperationType(InstCombiner &IC, LoadInst &LI) { return &LI; } - // Try to combine integer loads into vector loads when the integer is just - // loading a bag of bits that are casted into vector element chunks. - if (LI.getType()->isIntegerTy()) - if (Instruction *R = combineIntegerLoadToVectorLoad(IC, LI)) - return R; + // FIXME: We should also canonicalize loads of vectors when their elements are + // cast to other types. + return nullptr; +} + +// If we can determine that all possible objects pointed to by the provided +// pointer value are, not only dereferenceable, but also definitively less than +// or equal to the provided maximum size, then return true. Otherwise, return +// false (constant global values and allocas fall into this category). +// +// FIXME: This should probably live in ValueTracking (or similar). +static bool isObjectSizeLessThanOrEq(Value *V, uint64_t MaxSize, + const DataLayout &DL) { + SmallPtrSet Visited; + SmallVector Worklist(1, V); + + do { + Value *P = Worklist.pop_back_val(); + P = P->stripPointerCasts(); + + if (!Visited.insert(P).second) + continue; + + if (SelectInst *SI = dyn_cast(P)) { + Worklist.push_back(SI->getTrueValue()); + Worklist.push_back(SI->getFalseValue()); + continue; + } + + if (PHINode *PN = dyn_cast(P)) { + for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) + Worklist.push_back(PN->getIncomingValue(i)); + continue; + } + + if (GlobalAlias *GA = dyn_cast(P)) { + if (GA->mayBeOverridden()) + return false; + Worklist.push_back(GA->getAliasee()); + continue; + } + + // If we know how big this object is, and it is less than MaxSize, continue + // searching. Otherwise, return false. + if (AllocaInst *AI = dyn_cast(P)) { + if (!AI->getAllocatedType()->isSized()) + return false; + + ConstantInt *CS = dyn_cast(AI->getArraySize()); + if (!CS) + return false; + + uint64_t TypeSize = DL.getTypeAllocSize(AI->getAllocatedType()); + // Make sure that, even if the multiplication below would wrap as an + // uint64_t, we still do the right thing. + if ((CS->getValue().zextOrSelf(128)*APInt(128, TypeSize)).ugt(MaxSize)) + return false; + continue; + } + + if (GlobalVariable *GV = dyn_cast(P)) { + if (!GV->hasDefinitiveInitializer() || !GV->isConstant()) + return false; + + uint64_t InitSize = DL.getTypeAllocSize(GV->getType()->getElementType()); + if (InitSize > MaxSize) + return false; + continue; + } + + return false; + } while (!Worklist.empty()); + + return true; +} + +// If we're indexing into an object of a known size, and the outer index is +// not a constant, but having any value but zero would lead to undefined +// behavior, replace it with zero. +// +// For example, if we have: +// @f.a = private unnamed_addr constant [1 x i32] [i32 12], align 4 +// ... +// %arrayidx = getelementptr inbounds [1 x i32]* @f.a, i64 0, i64 %x +// ... = load i32* %arrayidx, align 4 +// Then we know that we can replace %x in the GEP with i64 0. +// +// FIXME: We could fold any GEP index to zero that would cause UB if it were +// not zero. Currently, we only handle the first such index. Also, we could +// also search through non-zero constant indices if we kept track of the +// offsets those indices implied. +static bool canReplaceGEPIdxWithZero(InstCombiner &IC, GetElementPtrInst *GEPI, + Instruction *MemI, unsigned &Idx) { + if (GEPI->getNumOperands() < 2) + return false; + + // Find the first non-zero index of a GEP. If all indices are zero, return + // one past the last index. + auto FirstNZIdx = [](const GetElementPtrInst *GEPI) { + unsigned I = 1; + for (unsigned IE = GEPI->getNumOperands(); I != IE; ++I) { + Value *V = GEPI->getOperand(I); + if (const ConstantInt *CI = dyn_cast(V)) + if (CI->isZero()) + continue; + + break; + } + + return I; + }; + + // Skip through initial 'zero' indices, and find the corresponding pointer + // type. See if the next index is not a constant. + Idx = FirstNZIdx(GEPI); + if (Idx == GEPI->getNumOperands()) + return false; + if (isa(GEPI->getOperand(Idx))) + return false; + + SmallVector Ops(GEPI->idx_begin(), GEPI->idx_begin() + Idx); + Type *AllocTy = + GetElementPtrInst::getIndexedType(GEPI->getOperand(0)->getType(), Ops); + if (!AllocTy || !AllocTy->isSized()) + return false; + const DataLayout &DL = IC.getDataLayout(); + uint64_t TyAllocSize = DL.getTypeAllocSize(AllocTy); + + // If there are more indices after the one we might replace with a zero, make + // sure they're all non-negative. If any of them are negative, the overall + // address being computed might be before the base address determined by the + // first non-zero index. + auto IsAllNonNegative = [&]() { + for (unsigned i = Idx+1, e = GEPI->getNumOperands(); i != e; ++i) { + bool KnownNonNegative, KnownNegative; + IC.ComputeSignBit(GEPI->getOperand(i), KnownNonNegative, + KnownNegative, 0, MemI); + if (KnownNonNegative) + continue; + return false; + } + + return true; + }; + + // FIXME: If the GEP is not inbounds, and there are extra indices after the + // one we'll replace, those could cause the address computation to wrap + // (rendering the IsAllNonNegative() check below insufficient). We can do + // better, ignoring zero indicies (and other indicies we can prove small + // enough not to wrap). + if (Idx+1 != GEPI->getNumOperands() && !GEPI->isInBounds()) + return false; + + // Note that isObjectSizeLessThanOrEq will return true only if the pointer is + // also known to be dereferenceable. + return isObjectSizeLessThanOrEq(GEPI->getOperand(0), TyAllocSize, DL) && + IsAllNonNegative(); +} + +// If we're indexing into an object with a variable index for the memory +// access, but the object has only one element, we can assume that the index +// will always be zero. If we replace the GEP, return it. +template +static Instruction *replaceGEPIdxWithZero(InstCombiner &IC, Value *Ptr, + T &MemI) { + if (GetElementPtrInst *GEPI = dyn_cast(Ptr)) { + unsigned Idx; + if (canReplaceGEPIdxWithZero(IC, GEPI, &MemI, Idx)) { + Instruction *NewGEPI = GEPI->clone(); + NewGEPI->setOperand(Idx, + ConstantInt::get(GEPI->getOperand(Idx)->getType(), 0)); + NewGEPI->insertBefore(GEPI); + MemI.setOperand(MemI.getPointerOperandIndex(), NewGEPI); + return NewGEPI; + } + } return nullptr; } @@ -539,18 +660,21 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) { return Res; // Attempt to improve the alignment. - if (DL) { - unsigned KnownAlign = - getOrEnforceKnownAlignment(Op, DL->getPrefTypeAlignment(LI.getType()), - DL, AT, &LI, DT); - unsigned LoadAlign = LI.getAlignment(); - unsigned EffectiveLoadAlign = LoadAlign != 0 ? LoadAlign : - DL->getABITypeAlignment(LI.getType()); - - if (KnownAlign > EffectiveLoadAlign) - LI.setAlignment(KnownAlign); - else if (LoadAlign == 0) - LI.setAlignment(EffectiveLoadAlign); + unsigned KnownAlign = getOrEnforceKnownAlignment( + Op, DL.getPrefTypeAlignment(LI.getType()), DL, &LI, AC, DT); + unsigned LoadAlign = LI.getAlignment(); + unsigned EffectiveLoadAlign = + LoadAlign != 0 ? LoadAlign : DL.getABITypeAlignment(LI.getType()); + + if (KnownAlign > EffectiveLoadAlign) + LI.setAlignment(KnownAlign); + else if (LoadAlign == 0) + LI.setAlignment(EffectiveLoadAlign); + + // Replace GEP indices if possible. + if (Instruction *NewGEPI = replaceGEPIdxWithZero(*this, Op, LI)) { + Worklist.Add(NewGEPI); + return &LI; } // None of the following transforms are legal for volatile/atomic loads. @@ -607,8 +731,8 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) { if (SelectInst *SI = dyn_cast(Op)) { // load (select (Cond, &V1, &V2)) --> select(Cond, load &V1, load &V2). unsigned Align = LI.getAlignment(); - if (isSafeToLoadUnconditionally(SI->getOperand(1), SI, Align, DL) && - isSafeToLoadUnconditionally(SI->getOperand(2), SI, Align, DL)) { + if (isSafeToLoadUnconditionally(SI->getOperand(1), SI, Align) && + isSafeToLoadUnconditionally(SI->getOperand(2), SI, Align)) { LoadInst *V1 = Builder->CreateLoad(SI->getOperand(1), SI->getOperand(1)->getName()+".val"); LoadInst *V2 = Builder->CreateLoad(SI->getOperand(2), @@ -619,218 +743,23 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) { } // load (select (cond, null, P)) -> load P - if (Constant *C = dyn_cast(SI->getOperand(1))) - if (C->isNullValue()) { - LI.setOperand(0, SI->getOperand(2)); - return &LI; - } + if (isa(SI->getOperand(1)) && + LI.getPointerAddressSpace() == 0) { + LI.setOperand(0, SI->getOperand(2)); + return &LI; + } // load (select (cond, P, null)) -> load P - if (Constant *C = dyn_cast(SI->getOperand(2))) - if (C->isNullValue()) { - LI.setOperand(0, SI->getOperand(1)); - return &LI; - } + if (isa(SI->getOperand(2)) && + LI.getPointerAddressSpace() == 0) { + LI.setOperand(0, SI->getOperand(1)); + return &LI; + } } } return nullptr; } -/// \brief Helper to combine a store to use a new value. -/// -/// This just does the work of combining a store to use a new value, potentially -/// of a different type. It handles metadata, etc., and returns the new -/// instruction. The new value is stored to a bitcast of the pointer argument to -/// the original store. -/// -/// Note that this will create the instructions with whatever insert point the -/// \c InstCombiner currently is using. -static StoreInst *combineStoreToNewValue(InstCombiner &IC, StoreInst &OldSI, - Value *V) { - Value *Ptr = OldSI.getPointerOperand(); - unsigned AS = OldSI.getPointerAddressSpace(); - SmallVector, 8> MD; - OldSI.getAllMetadata(MD); - - StoreInst *NewSI = IC.Builder->CreateAlignedStore( - V, IC.Builder->CreateBitCast(Ptr, V->getType()->getPointerTo(AS)), - OldSI.getAlignment()); - for (const auto &MDPair : MD) { - unsigned ID = MDPair.first; - MDNode *N = MDPair.second; - // Note, essentially every kind of metadata should be preserved here! This - // routine is supposed to clone a store instruction changing *only its - // type*. The only metadata it makes sense to drop is metadata which is - // invalidated when the pointer type changes. This should essentially - // never be the case in LLVM, but we explicitly switch over only known - // metadata to be conservatively correct. If you are adding metadata to - // LLVM which pertains to stores, you almost certainly want to add it - // here. - switch (ID) { - case LLVMContext::MD_dbg: - case LLVMContext::MD_tbaa: - case LLVMContext::MD_prof: - case LLVMContext::MD_fpmath: - case LLVMContext::MD_tbaa_struct: - case LLVMContext::MD_alias_scope: - case LLVMContext::MD_noalias: - case LLVMContext::MD_nontemporal: - case LLVMContext::MD_mem_parallel_loop_access: - case LLVMContext::MD_nonnull: - // All of these directly apply. - NewSI->setMetadata(ID, N); - break; - - case LLVMContext::MD_invariant_load: - case LLVMContext::MD_range: - break; - } - } - return NewSI; -} - -/// \brief Combine integer stores to vector stores when the integers bits are -/// just a concatenation of non-integer (and non-vector) types. -/// -/// This specifically matches the pattern of taking a sequence of non-integer -/// types, casting them to integers, extending, shifting, and or-ing them -/// together to make a concatenation, and then storing the result. This shows up -/// because large integers are sometimes used to represent a "generic" load or -/// store, and only later optimization may uncover that there is a more natural -/// type to represent the store with. -/// -/// \returns true if the store was successfully combined away. This indicates -/// the caller must erase the store instruction. We have to let the caller erase -/// the store instruction sas otherwise there is no way to signal whether it was -/// combined or not: IC.EraseInstFromFunction returns a null pointer. -static bool combineIntegerStoreToVectorStore(InstCombiner &IC, StoreInst &SI) { - // FIXME: This is probably a reasonable transform to make for atomic stores. - assert(SI.isSimple() && "Do not call for non-simple stores!"); - - Instruction *OrigV = dyn_cast(SI.getValueOperand()); - if (!OrigV) - return false; - - // We only handle values which are used entirely to store to memory. If the - // value is used directly as an SSA value, then even if there are matching - // element insertion and element extraction, we rely on basic integer - // combining to forward the bits and delete the intermediate math. Here we - // just need to clean up the places where it actually reaches memory. - SmallVector Stores; - for (User *U : OrigV->users()) - if (auto *SIU = dyn_cast(U)) - Stores.push_back(SIU); - else - return false; - - const DataLayout &DL = *IC.getDataLayout(); - unsigned BaseBits = OrigV->getType()->getIntegerBitWidth(); - Type *ElementTy = nullptr; - int ElementSize; - - // We need to match some number of element insertions into an integer. Each - // insertion takes the form of an element value (and type), index (multiple of - // the bitwidth of the type) of insertion, and the base it was inserted into. - struct InsertedElement { - Value *Base; - Value *Element; - int Index; - }; - auto MatchInsertedElement = [&](Value *V) -> Optional { - // Handle a null input to make it easy to loop over bases. - if (!V) - return Optional(); - - assert(!V->getType()->isVectorTy() && "Must not be a vector."); - assert(V->getType()->isIntegerTy() && "Must be an integer value."); - - Value *Base = nullptr, *Cast; - ConstantInt *ShiftC = nullptr; - auto InsertPattern = m_CombineOr( - m_Shl(m_OneUse(m_ZExt(m_OneUse(m_Value(Cast)))), m_ConstantInt(ShiftC)), - m_ZExt(m_OneUse(m_Value(Cast)))); - if (!match(V, m_CombineOr(m_CombineOr(m_Or(m_OneUse(m_Value(Base)), - m_OneUse(InsertPattern)), - m_Or(m_OneUse(InsertPattern), - m_OneUse(m_Value(Base)))), - InsertPattern))) - return Optional(); - - Value *Element; - if (auto *BC = dyn_cast(Cast)) { - // Bit casts are trivially correct here. - Element = BC->getOperand(0); - } else if (auto *PC = dyn_cast(Cast)) { - Element = PC->getOperand(0); - // If this changes the bit width at all, reject it. - if (PC->getType()->getIntegerBitWidth() != - DL.getTypeSizeInBits(Element->getType())) - return Optional(); - } else { - // All other casts are rejected. - return Optional(); - } - - // We can't handle shifts wider than the number of bits in the integer. - unsigned ShiftBits = ShiftC ? ShiftC->getLimitedValue(BaseBits) : 0; - if (ShiftBits == BaseBits) - return Optional(); - - // All of the elements inserted need to be the same type. Either capture the - // first element type or check this element type against the previous - // element types. - if (!ElementTy) { - ElementTy = Element->getType(); - // The base integer size and the shift need to be multiples of the element - // size in bits. - ElementSize = DL.getTypeSizeInBits(ElementTy); - if (BaseBits % ElementSize || ShiftBits % ElementSize) - return Optional(); - } else if (ElementTy != Element->getType()) { - return Optional(); - } - - // We don't handle integers, sub-vectors, or any aggregate types. We - // handle pointers and floating ponit types. - if (!ElementTy->isSingleValueType() || ElementTy->isIntegerTy() || - ElementTy->isVectorTy()) - return Optional(); - - int Index = - (DL.isLittleEndian() ? ShiftBits : BaseBits - ElementSize - ShiftBits) / - ElementSize; - InsertedElement Result = {Base, Element, Index}; - return Result; - }; - - SmallVector Elements; - Value *V = OrigV; - while (Optional E = MatchInsertedElement(V)) { - V = E->Base; - Elements.push_back(std::move(*E)); - } - // If searching for elements found none, or didn't terminate in either an - // undef or a direct zext, we can't form a vector. - if (Elements.empty() || (V && !isa(V))) - return false; - - // Build a storable vector by looping over the inserted elements. - VectorType *VTy = VectorType::get(ElementTy, BaseBits / ElementSize); - V = UndefValue::get(VTy); - IC.Builder->SetInsertPoint(OrigV); - for (const auto &E : Elements) - V = IC.Builder->CreateInsertElement(V, E.Element, - IC.Builder->getInt32(E.Index)); - - for (StoreInst *OldSI : Stores) { - IC.Builder->SetInsertPoint(OldSI); - combineStoreToNewValue(IC, *OldSI, V); - if (OldSI != &SI) - IC.EraseInstFromFunction(*OldSI); - } - return true; -} - /// \brief Combine stores to match the type of value being stored. /// /// The core idea here is that the memory does not have any intrinsic type and @@ -861,16 +790,11 @@ static bool combineStoreToValueType(InstCombiner &IC, StoreInst &SI) { // Fold away bit casts of the stored value by storing the original type. if (auto *BC = dyn_cast(V)) { - combineStoreToNewValue(IC, SI, BC->getOperand(0)); + V = BC->getOperand(0); + combineStoreToNewValue(IC, SI, V); return true; } - // If this is an integer store and we have data layout, look for a pattern of - // storing a vector as an integer (modeled as a bag of bits). - if (V->getType()->isIntegerTy() && IC.getDataLayout() && - combineIntegerStoreToVectorStore(IC, SI)) - return true; - // FIXME: We should also canonicalize loads of vectors when their elements are // cast to other types. return false; @@ -914,18 +838,21 @@ Instruction *InstCombiner::visitStoreInst(StoreInst &SI) { return EraseInstFromFunction(SI); // Attempt to improve the alignment. - if (DL) { - unsigned KnownAlign = - getOrEnforceKnownAlignment(Ptr, DL->getPrefTypeAlignment(Val->getType()), - DL, AT, &SI, DT); - unsigned StoreAlign = SI.getAlignment(); - unsigned EffectiveStoreAlign = StoreAlign != 0 ? StoreAlign : - DL->getABITypeAlignment(Val->getType()); - - if (KnownAlign > EffectiveStoreAlign) - SI.setAlignment(KnownAlign); - else if (StoreAlign == 0) - SI.setAlignment(EffectiveStoreAlign); + unsigned KnownAlign = getOrEnforceKnownAlignment( + Ptr, DL.getPrefTypeAlignment(Val->getType()), DL, &SI, AC, DT); + unsigned StoreAlign = SI.getAlignment(); + unsigned EffectiveStoreAlign = + StoreAlign != 0 ? StoreAlign : DL.getABITypeAlignment(Val->getType()); + + if (KnownAlign > EffectiveStoreAlign) + SI.setAlignment(KnownAlign); + else if (StoreAlign == 0) + SI.setAlignment(EffectiveStoreAlign); + + // Replace GEP indices if possible. + if (Instruction *NewGEPI = replaceGEPIdxWithZero(*this, Ptr, SI)) { + Worklist.Add(NewGEPI); + return &SI; } // Don't hack volatile/atomic stores.