X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FTransforms%2FInstrumentation%2FMemorySanitizer.cpp;h=65db206e570d86668e9432b66f044812b20946f5;hb=63799f6febc91ebec0d308737bfd1c659e4c24b7;hp=6407740065a9ff2f40e1a9bc3c522f3c8adc7f0b;hpb=33660cdfbd521f39982e86844db6784848b8f5d5;p=oota-llvm.git diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 6407740065a..65db206e570 100644 --- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -43,32 +43,82 @@ /// parameters and return values may be passed via registers, we have a /// specialized thread-local shadow for return values /// (__msan_retval_tls) and parameters (__msan_param_tls). +/// +/// Origin tracking. +/// +/// MemorySanitizer can track origins (allocation points) of all uninitialized +/// values. This behavior is controlled with a flag (msan-track-origins) and is +/// disabled by default. +/// +/// Origins are 4-byte values created and interpreted by the runtime library. +/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes +/// of application memory. Propagation of origins is basically a bunch of +/// "select" instructions that pick the origin of a dirty argument, if an +/// instruction has one. +/// +/// Every 4 aligned, consecutive bytes of application memory have one origin +/// value associated with them. If these bytes contain uninitialized data +/// coming from 2 different allocations, the last store wins. Because of this, +/// MemorySanitizer reports can show unrelated origins, but this is unlikely in +/// practice. +/// +/// Origins are meaningless for fully initialized values, so MemorySanitizer +/// avoids storing origin to memory when a fully initialized value is stored. +/// This way it avoids needless overwritting origin of the 4-byte region on +/// a short (i.e. 1 byte) clean store, and it is also good for performance. +/// +/// Atomic handling. +/// +/// Ideally, every atomic store of application value should update the +/// corresponding shadow location in an atomic way. Unfortunately, atomic store +/// of two disjoint locations can not be done without severe slowdown. +/// +/// Therefore, we implement an approximation that may err on the safe side. +/// In this implementation, every atomically accessed location in the program +/// may only change from (partially) uninitialized to fully initialized, but +/// not the other way around. We load the shadow _after_ the application load, +/// and we store the shadow _before_ the app store. Also, we always store clean +/// shadow (if the application store is atomic). This way, if the store-load +/// pair constitutes a happens-before arc, shadow store and load are correctly +/// ordered such that the load will get either the value that was stored, or +/// some later value (which is always clean). +/// +/// This does not work very well with Compare-And-Swap (CAS) and +/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW +/// must store the new shadow before the app operation, and load the shadow +/// after the app operation. Computers don't work this way. Current +/// implementation ignores the load aspect of CAS/RMW, always returning a clean +/// value. It implements the store part as a simple atomic store by storing a +/// clean shadow. + //===----------------------------------------------------------------------===// #define DEBUG_TYPE "msan" #include "llvm/Transforms/Instrumentation.h" -#include "BlackList.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Triple.h" #include "llvm/ADT/ValueMap.h" -#include "llvm/DataLayout.h" -#include "llvm/Function.h" -#include "llvm/IRBuilder.h" -#include "llvm/InlineAsm.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" #include "llvm/InstVisitor.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" -#include "llvm/MDBuilder.h" -#include "llvm/Module.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/ModuleUtils.h" -#include "llvm/Type.h" +#include "llvm/Transforms/Utils/SpecialCaseList.h" using namespace llvm; @@ -76,13 +126,13 @@ static const uint64_t kShadowMask32 = 1ULL << 31; static const uint64_t kShadowMask64 = 1ULL << 46; static const uint64_t kOriginOffset32 = 1ULL << 30; static const uint64_t kOriginOffset64 = 1ULL << 45; -static const uint64_t kShadowTLSAlignment = 8; +static const unsigned kMinOriginAlignment = 4; +static const unsigned kShadowTLSAlignment = 8; -// This is an important flag that makes the reports much more -// informative at the cost of greater slowdown. Not fully implemented -// yet. -// FIXME: this should be a top-level clang flag, e.g. -// -fmemory-sanitizer-full. +/// \brief Track origins of uninitialized values. +/// +/// Adds a section to MemorySanitizer report that points to the allocation +/// (stack or heap) the uninitialized bits came from originally. static cl::opt ClTrackOrigins("msan-track-origins", cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden, cl::init(false)); @@ -98,11 +148,18 @@ static cl::opt ClPoisonStackWithCall("msan-poison-stack-with-call", static cl::opt ClPoisonStackPattern("msan-poison-stack-pattern", cl::desc("poison uninitialized stack variables with the given patter"), cl::Hidden, cl::init(0xff)); +static cl::opt ClPoisonUndef("msan-poison-undef", + cl::desc("poison undef temps"), + cl::Hidden, cl::init(true)); static cl::opt ClHandleICmp("msan-handle-icmp", cl::desc("propagate shadow through ICmpEQ and ICmpNE"), cl::Hidden, cl::init(true)); +static cl::opt ClHandleICmpExact("msan-handle-icmp-exact", + cl::desc("exact handling of relational integer ICmp"), + cl::Hidden, cl::init(false)); + static cl::opt ClStoreCleanOrigin("msan-store-clean-origin", cl::desc("store origin for clean (fully initialized) values"), cl::Hidden, cl::init(false)); @@ -121,10 +178,18 @@ static cl::opt ClDumpStrictInstructions("msan-dump-strict-instructions", cl::desc("print out instructions with default strict semantics"), cl::Hidden, cl::init(false)); -static cl::opt ClBlackListFile("msan-blacklist", +static cl::opt ClBlacklistFile("msan-blacklist", cl::desc("File containing the list of functions where MemorySanitizer " "should not report bugs"), cl::Hidden); +// Experimental. Wraps all indirect calls in the instrumented code with +// a call to the given function. This is needed to assist the dynamic +// helper tool (MSanDR) to regain control on transition between instrumented and +// non-instrumented code. +static cl::opt ClWrapIndirectCalls("msan-wrap-indirect-calls", + cl::desc("Wrap indirect calls with a given function"), + cl::Hidden); + namespace { /// \brief An instrumentation pass implementing detection of uninitialized @@ -134,11 +199,14 @@ namespace { /// uninitialized reads. class MemorySanitizer : public FunctionPass { public: - MemorySanitizer(bool TrackOrigins = false) - : FunctionPass(ID), - TrackOrigins(TrackOrigins || ClTrackOrigins), - TD(0), - WarningFn(0) { } + MemorySanitizer(bool TrackOrigins = false, + StringRef BlacklistFile = StringRef()) + : FunctionPass(ID), + TrackOrigins(TrackOrigins || ClTrackOrigins), + TD(0), + WarningFn(0), + BlacklistFile(BlacklistFile.empty() ? ClBlacklistFile : BlacklistFile), + WrapIndirectCalls(!ClWrapIndirectCalls.empty()) {} const char *getPassName() const { return "MemorySanitizer"; } bool runOnFunction(Function &F); bool doInitialization(Module &M); @@ -178,7 +246,7 @@ class MemorySanitizer : public FunctionPass { Value *MsanCopyOriginFn; /// \brief Run-time helper that generates a new origin value for a stack /// allocation. - Value *MsanSetAllocaOriginFn; + Value *MsanSetAllocaOrigin4Fn; /// \brief Run-time helper that poisons stack on function entry. Value *MsanPoisonStackFn; /// \brief MSan runtime replacements for memmove, memcpy and memset. @@ -194,11 +262,19 @@ class MemorySanitizer : public FunctionPass { MDNode *ColdCallWeights; /// \brief Branch weights for origin store. MDNode *OriginStoreWeights; + /// \brief Path to blacklist file. + SmallString<64> BlacklistFile; /// \brief The blacklist. - OwningPtr BL; + OwningPtr BL; /// \brief An empty volatile inline asm that prevents callback merge. InlineAsm *EmptyAsm; + bool WrapIndirectCalls; + /// \brief Run-time wrapper for indirect calls. + Value *IndirectCallWrapperFn; + // Argument and return type of IndirectCallWrapperFn: void (*f)(void). + Type *AnyFunctionPtrTy; + friend struct MemorySanitizerVisitor; friend struct VarArgAMD64Helper; }; @@ -209,8 +285,9 @@ INITIALIZE_PASS(MemorySanitizer, "msan", "MemorySanitizer: detects uninitialized reads.", false, false) -FunctionPass *llvm::createMemorySanitizerPass(bool TrackOrigins) { - return new MemorySanitizer(TrackOrigins); +FunctionPass *llvm::createMemorySanitizerPass(bool TrackOrigins, + StringRef BlacklistFile) { + return new MemorySanitizer(TrackOrigins, BlacklistFile); } /// \brief Create a non-const global initialized with the given string. @@ -243,9 +320,9 @@ void MemorySanitizer::initializeCallbacks(Module &M) { MsanCopyOriginFn = M.getOrInsertFunction( "__msan_copy_origin", IRB.getVoidTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, NULL); - MsanSetAllocaOriginFn = M.getOrInsertFunction( - "__msan_set_alloca_origin", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, - IRB.getInt8PtrTy(), NULL); + MsanSetAllocaOrigin4Fn = M.getOrInsertFunction( + "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, + IRB.getInt8PtrTy(), IntptrTy, NULL); MsanPoisonStackFn = M.getOrInsertFunction( "__msan_poison_stack", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, NULL); MemmoveFn = M.getOrInsertFunction( @@ -262,35 +339,42 @@ void MemorySanitizer::initializeCallbacks(Module &M) { RetvalTLS = new GlobalVariable( M, ArrayType::get(IRB.getInt64Ty(), 8), false, GlobalVariable::ExternalLinkage, 0, "__msan_retval_tls", 0, - GlobalVariable::GeneralDynamicTLSModel); + GlobalVariable::InitialExecTLSModel); RetvalOriginTLS = new GlobalVariable( M, OriginTy, false, GlobalVariable::ExternalLinkage, 0, - "__msan_retval_origin_tls", 0, GlobalVariable::GeneralDynamicTLSModel); + "__msan_retval_origin_tls", 0, GlobalVariable::InitialExecTLSModel); ParamTLS = new GlobalVariable( M, ArrayType::get(IRB.getInt64Ty(), 1000), false, GlobalVariable::ExternalLinkage, 0, "__msan_param_tls", 0, - GlobalVariable::GeneralDynamicTLSModel); + GlobalVariable::InitialExecTLSModel); ParamOriginTLS = new GlobalVariable( M, ArrayType::get(OriginTy, 1000), false, GlobalVariable::ExternalLinkage, - 0, "__msan_param_origin_tls", 0, GlobalVariable::GeneralDynamicTLSModel); + 0, "__msan_param_origin_tls", 0, GlobalVariable::InitialExecTLSModel); VAArgTLS = new GlobalVariable( M, ArrayType::get(IRB.getInt64Ty(), 1000), false, GlobalVariable::ExternalLinkage, 0, "__msan_va_arg_tls", 0, - GlobalVariable::GeneralDynamicTLSModel); + GlobalVariable::InitialExecTLSModel); VAArgOverflowSizeTLS = new GlobalVariable( M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, 0, "__msan_va_arg_overflow_size_tls", 0, - GlobalVariable::GeneralDynamicTLSModel); + GlobalVariable::InitialExecTLSModel); OriginTLS = new GlobalVariable( M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, 0, - "__msan_origin_tls", 0, GlobalVariable::GeneralDynamicTLSModel); + "__msan_origin_tls", 0, GlobalVariable::InitialExecTLSModel); // We insert an empty inline asm after __msan_report* to avoid callback merge. EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), StringRef(""), StringRef(""), /*hasSideEffects=*/true); + + if (WrapIndirectCalls) { + AnyFunctionPtrTy = + PointerType::getUnqual(FunctionType::get(IRB.getVoidTy(), false)); + IndirectCallWrapperFn = M.getOrInsertFunction( + ClWrapIndirectCalls, AnyFunctionPtrTy, AnyFunctionPtrTy, NULL); + } } /// \brief Module-level initialization. @@ -300,7 +384,7 @@ bool MemorySanitizer::doInitialization(Module &M) { TD = getAnalysisIfAvailable(); if (!TD) return false; - BL.reset(new BlackList(ClBlackListFile)); + BL.reset(SpecialCaseList::createOrDie(BlacklistFile)); C = &(M.getContext()); unsigned PtrSize = TD->getPointerSizeInBits(/* AddressSpace */0); switch (PtrSize) { @@ -328,8 +412,13 @@ bool MemorySanitizer::doInitialization(Module &M) { appendToGlobalCtors(M, cast(M.getOrInsertFunction( "__msan_init", IRB.getVoidTy(), NULL)), 0); - new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, - IRB.getInt32(TrackOrigins), "__msan_track_origins"); + if (TrackOrigins) + new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, + IRB.getInt32(TrackOrigins), "__msan_track_origins"); + + if (ClKeepGoing) + new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, + IRB.getInt32(ClKeepGoing), "__msan_keep_going"); return true; } @@ -381,13 +470,12 @@ struct MemorySanitizerVisitor : public InstVisitor { SmallVector ShadowPHINodes, OriginPHINodes; ValueMap ShadowMap, OriginMap; bool InsertChecks; + bool LoadShadow; + bool PoisonStack; + bool PoisonUndef; + bool CheckReturnValue; OwningPtr VAHelper; - // An unfortunate workaround for asymmetric lowering of va_arg stuff. - // See a comment in visitCallSite for more details. - static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7 - static const unsigned AMD64FpEndOffset = 176; - struct ShadowOriginAndInsertPoint { Instruction *Shadow; Instruction *Origin; @@ -400,11 +488,21 @@ struct MemorySanitizerVisitor : public InstVisitor { SmallVector StoreList; MemorySanitizerVisitor(Function &F, MemorySanitizer &MS) - : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) { - InsertChecks = !MS.BL->isIn(F); + : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) { + bool SanitizeFunction = !MS.BL->isIn(F) && F.getAttributes().hasAttribute( + AttributeSet::FunctionIndex, + Attribute::SanitizeMemory); + InsertChecks = SanitizeFunction; + LoadShadow = SanitizeFunction; + PoisonStack = SanitizeFunction && ClPoisonStack; + PoisonUndef = SanitizeFunction && ClPoisonUndef; + // FIXME: Consider using SpecialCaseList to specify a list of functions that + // must always return fully initialized values. For now, we hardcode "main". + CheckReturnValue = SanitizeFunction && (F.getName() == "main"); + DEBUG(if (!InsertChecks) - dbgs() << "MemorySanitizer is not inserting checks into '" - << F.getName() << "'\n"); + dbgs() << "MemorySanitizer is not inserting checks into '" + << F.getName() << "'\n"); } void materializeStores() { @@ -414,22 +512,25 @@ struct MemorySanitizerVisitor : public InstVisitor { IRBuilder<> IRB(&I); Value *Val = I.getValueOperand(); Value *Addr = I.getPointerOperand(); - Value *Shadow = getShadow(Val); + Value *Shadow = I.isAtomic() ? getCleanShadow(Val) : getShadow(Val); Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB); StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, I.getAlignment()); DEBUG(dbgs() << " STORE: " << *NewSI << "\n"); (void)NewSI; - // If the store is volatile, add a check. - if (I.isVolatile()) - insertCheck(Val, &I); + if (ClCheckAccessAddress) insertCheck(Addr, &I); + if (I.isAtomic()) + I.setOrdering(addReleaseOrdering(I.getOrdering())); + if (MS.TrackOrigins) { + unsigned Alignment = std::max(kMinOriginAlignment, I.getAlignment()); if (ClStoreCleanOrigin || isa(Shadow->getType())) { - IRB.CreateStore(getOrigin(Val), getOriginPtr(Addr, IRB)); + IRB.CreateAlignedStore(getOrigin(Val), getOriginPtr(Addr, IRB), + Alignment); } else { Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB); @@ -446,7 +547,8 @@ struct MemorySanitizerVisitor : public InstVisitor { SplitBlockAndInsertIfThen(cast(Cmp), false, MS.OriginStoreWeights); IRBuilder<> IRBNew(CheckTerm); - IRBNew.CreateStore(getOrigin(Val), getOriginPtr(Addr, IRBNew)); + IRBNew.CreateAlignedStore(getOrigin(Val), getOriginPtr(Addr, IRBNew), + Alignment); } } } @@ -485,6 +587,13 @@ struct MemorySanitizerVisitor : public InstVisitor { bool runOnFunction() { MS.initializeCallbacks(*F.getParent()); if (!MS.TD) return false; + + // In the presence of unreachable blocks, we may see Phi nodes with + // incoming nodes from such blocks. Since InstVisitor skips unreachable + // blocks, such nodes will not have any shadow value associated with them. + // It's easier to remove unreachable blocks than deal with missing shadow. + removeUnreachableBlocks(F); + // Iterate all BBs in depth-first order and create shadow instructions // for all instructions (where applicable). // For PHI nodes we create dummy shadow PHIs which will be finalized later. @@ -533,8 +642,11 @@ struct MemorySanitizerVisitor : public InstVisitor { // This may return weird-sized types like i1. if (IntegerType *IT = dyn_cast(OrigTy)) return IT; - if (VectorType *VT = dyn_cast(OrigTy)) - return VectorType::getInteger(VT); + if (VectorType *VT = dyn_cast(OrigTy)) { + uint32_t EltSize = MS.TD->getTypeSizeInBits(VT->getElementType()); + return VectorType::get(IntegerType::get(*MS.C, EltSize), + VT->getNumElements()); + } if (StructType *ST = dyn_cast(OrigTy)) { SmallVector Elements; for (unsigned i = 0, n = ST->getNumElements(); i < n; i++) @@ -543,7 +655,7 @@ struct MemorySanitizerVisitor : public InstVisitor { DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n"); return Res; } - uint32_t TypeSize = MS.TD->getTypeStoreSizeInBits(OrigTy); + uint32_t TypeSize = MS.TD->getTypeSizeInBits(OrigTy); return IntegerType::get(*MS.C, TypeSize); } @@ -642,7 +754,7 @@ struct MemorySanitizerVisitor : public InstVisitor { /// /// Clean shadow (all zeroes) means all bits of the value are defined /// (initialized). - Value *getCleanShadow(Value *V) { + Constant *getCleanShadow(Value *V) { Type *ShadowTy = getShadowTy(V); if (!ShadowTy) return 0; @@ -661,6 +773,14 @@ struct MemorySanitizerVisitor : public InstVisitor { return ConstantStruct::get(ST, Vals); } + /// \brief Create a dirty shadow for a given value. + Constant *getPoisonedShadow(Value *V) { + Type *ShadowTy = getShadowTy(V); + if (!ShadowTy) + return 0; + return getPoisonedShadow(ShadowTy); + } + /// \brief Create a clean (zero) origin. Value *getCleanOrigin() { return Constant::getNullValue(MS.OriginTy); @@ -682,7 +802,7 @@ struct MemorySanitizerVisitor : public InstVisitor { return Shadow; } if (UndefValue *U = dyn_cast(V)) { - Value *AllOnes = getPoisonedShadow(getShadowTy(V)); + Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V); DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n"); (void)U; return AllOnes; @@ -709,14 +829,21 @@ struct MemorySanitizerVisitor : public InstVisitor { if (AI->hasByValAttr()) { // ByVal pointer itself has clean shadow. We copy the actual // argument shadow to the underlying memory. + // Figure out maximal valid memcpy alignment. + unsigned ArgAlign = AI->getParamAlignment(); + if (ArgAlign == 0) { + Type *EltType = A->getType()->getPointerElementType(); + ArgAlign = MS.TD->getABITypeAlignment(EltType); + } + unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment); Value *Cpy = EntryIRB.CreateMemCpy( - getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), - Base, Size, AI->getParamAlignment()); + getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size, + CopyAlign); DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n"); (void)Cpy; *ShadowPtr = getCleanShadow(V); } else { - *ShadowPtr = EntryIRB.CreateLoad(Base); + *ShadowPtr = EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment); } DEBUG(dbgs() << " ARG: " << *AI << " ==> " << **ShadowPtr << "\n"); @@ -725,7 +852,7 @@ struct MemorySanitizerVisitor : public InstVisitor { setOrigin(A, EntryIRB.CreateLoad(OriginPtr)); } } - ArgOffset += DataLayout::RoundUpAlignment(Size, 8); + ArgOffset += DataLayout::RoundUpAlignment(Size, kShadowTLSAlignment); } assert(*ShadowPtr && "Could not find shadow for an argument"); return *ShadowPtr; @@ -777,6 +904,40 @@ struct MemorySanitizerVisitor : public InstVisitor { ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns)); } + AtomicOrdering addReleaseOrdering(AtomicOrdering a) { + switch (a) { + case NotAtomic: + return NotAtomic; + case Unordered: + case Monotonic: + case Release: + return Release; + case Acquire: + case AcquireRelease: + return AcquireRelease; + case SequentiallyConsistent: + return SequentiallyConsistent; + } + llvm_unreachable("Unknown ordering"); + } + + AtomicOrdering addAcquireOrdering(AtomicOrdering a) { + switch (a) { + case NotAtomic: + return NotAtomic; + case Unordered: + case Monotonic: + case Acquire: + return Acquire; + case Release: + case AcquireRelease: + return AcquireRelease; + case SequentiallyConsistent: + return SequentiallyConsistent; + } + llvm_unreachable("Unknown ordering"); + } + // ------------------- Visitors. /// \brief Instrument LoadInst @@ -785,28 +946,73 @@ struct MemorySanitizerVisitor : public InstVisitor { /// Optionally, checks that the load address is fully defined. void visitLoadInst(LoadInst &I) { assert(I.getType()->isSized() && "Load type must have size"); - IRBuilder<> IRB(&I); + IRBuilder<> IRB(I.getNextNode()); Type *ShadowTy = getShadowTy(&I); Value *Addr = I.getPointerOperand(); - Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB); - setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, I.getAlignment(), "_msld")); + if (LoadShadow) { + Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB); + setShadow(&I, + IRB.CreateAlignedLoad(ShadowPtr, I.getAlignment(), "_msld")); + } else { + setShadow(&I, getCleanShadow(&I)); + } if (ClCheckAccessAddress) insertCheck(I.getPointerOperand(), &I); - if (MS.TrackOrigins) - setOrigin(&I, IRB.CreateLoad(getOriginPtr(Addr, IRB))); + if (I.isAtomic()) + I.setOrdering(addAcquireOrdering(I.getOrdering())); + + if (MS.TrackOrigins) { + if (LoadShadow) { + unsigned Alignment = std::max(kMinOriginAlignment, I.getAlignment()); + setOrigin(&I, + IRB.CreateAlignedLoad(getOriginPtr(Addr, IRB), Alignment)); + } else { + setOrigin(&I, getCleanOrigin()); + } + } } /// \brief Instrument StoreInst /// /// Stores the corresponding shadow and (optionally) origin. /// Optionally, checks that the store address is fully defined. - /// Volatile stores check that the value being stored is fully defined. void visitStoreInst(StoreInst &I) { StoreList.push_back(&I); } + void handleCASOrRMW(Instruction &I) { + assert(isa(I) || isa(I)); + + IRBuilder<> IRB(&I); + Value *Addr = I.getOperand(0); + Value *ShadowPtr = getShadowPtr(Addr, I.getType(), IRB); + + if (ClCheckAccessAddress) + insertCheck(Addr, &I); + + // Only test the conditional argument of cmpxchg instruction. + // The other argument can potentially be uninitialized, but we can not + // detect this situation reliably without possible false positives. + if (isa(I)) + insertCheck(I.getOperand(1), &I); + + IRB.CreateStore(getCleanShadow(&I), ShadowPtr); + + setShadow(&I, getCleanShadow(&I)); + } + + void visitAtomicRMWInst(AtomicRMWInst &I) { + handleCASOrRMW(I); + I.setOrdering(addReleaseOrdering(I.getOrdering())); + } + + void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { + handleCASOrRMW(I); + I.setOrdering(addReleaseOrdering(I.getOrdering())); + } + // Vector manipulation. void visitExtractElementInst(ExtractElementInst &I) { insertCheck(I.getOperand(1), &I); @@ -1015,6 +1221,8 @@ struct MemorySanitizerVisitor : public InstVisitor { } size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) { + assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && + "Vector of pointers is not a valid shadow type"); return Ty->isVectorTy() ? Ty->getVectorNumElements() * Ty->getScalarSizeInBits() : Ty->getPrimitiveSizeInBits(); @@ -1080,10 +1288,13 @@ struct MemorySanitizerVisitor : public InstVisitor { Value *B = I.getOperand(1); Value *Sa = getShadow(A); Value *Sb = getShadow(B); - if (A->getType()->isPointerTy()) - A = IRB.CreatePointerCast(A, MS.IntptrTy); - if (B->getType()->isPointerTy()) - B = IRB.CreatePointerCast(B, MS.IntptrTy); + + // Get rid of pointers and vectors of pointers. + // For ints (and vectors of ints), types of A and Sa match, + // and this is a no-op. + A = IRB.CreatePointerCast(A, Sa->getType()); + B = IRB.CreatePointerCast(B, Sb->getType()); + // A == B <==> (C = A^B) == 0 // A != B <==> (C = A^B) != 0 // Sc = Sa | Sb @@ -1105,6 +1316,73 @@ struct MemorySanitizerVisitor : public InstVisitor { setOriginForNaryOp(I); } + /// \brief Build the lowest possible value of V, taking into account V's + /// uninitialized bits. + Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa, + bool isSigned) { + if (isSigned) { + // Split shadow into sign bit and other bits. + Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1); + Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits); + // Maximise the undefined shadow bit, minimize other undefined bits. + return + IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit); + } else { + // Minimize undefined bits. + return IRB.CreateAnd(A, IRB.CreateNot(Sa)); + } + } + + /// \brief Build the highest possible value of V, taking into account V's + /// uninitialized bits. + Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa, + bool isSigned) { + if (isSigned) { + // Split shadow into sign bit and other bits. + Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1); + Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits); + // Minimise the undefined shadow bit, maximise other undefined bits. + return + IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits); + } else { + // Maximize undefined bits. + return IRB.CreateOr(A, Sa); + } + } + + /// \brief Instrument relational comparisons. + /// + /// This function does exact shadow propagation for all relational + /// comparisons of integers, pointers and vectors of those. + /// FIXME: output seems suboptimal when one of the operands is a constant + void handleRelationalComparisonExact(ICmpInst &I) { + IRBuilder<> IRB(&I); + Value *A = I.getOperand(0); + Value *B = I.getOperand(1); + Value *Sa = getShadow(A); + Value *Sb = getShadow(B); + + // Get rid of pointers and vectors of pointers. + // For ints (and vectors of ints), types of A and Sa match, + // and this is a no-op. + A = IRB.CreatePointerCast(A, Sa->getType()); + B = IRB.CreatePointerCast(B, Sb->getType()); + + // Let [a0, a1] be the interval of possible values of A, taking into account + // its undefined bits. Let [b0, b1] be the interval of possible values of B. + // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0). + bool IsSigned = I.isSigned(); + Value *S1 = IRB.CreateICmp(I.getPredicate(), + getLowestPossibleValue(IRB, A, Sa, IsSigned), + getHighestPossibleValue(IRB, B, Sb, IsSigned)); + Value *S2 = IRB.CreateICmp(I.getPredicate(), + getHighestPossibleValue(IRB, A, Sa, IsSigned), + getLowestPossibleValue(IRB, B, Sb, IsSigned)); + Value *Si = IRB.CreateXor(S1, S2); + setShadow(&I, Si); + setOriginForNaryOp(I); + } + /// \brief Instrument signed relational comparisons. /// /// Handle (x<0) and (x>=0) comparisons (essentially, sign bit tests) by @@ -1134,12 +1412,32 @@ struct MemorySanitizerVisitor : public InstVisitor { } void visitICmpInst(ICmpInst &I) { - if (ClHandleICmp && I.isEquality()) + if (!ClHandleICmp) { + handleShadowOr(I); + return; + } + if (I.isEquality()) { handleEqualityComparison(I); - else if (ClHandleICmp && I.isSigned() && I.isRelational()) + return; + } + + assert(I.isRelational()); + if (ClHandleICmpExact) { + handleRelationalComparisonExact(I); + return; + } + if (I.isSigned()) { handleSignedRelationalComparison(I); - else - handleShadowOr(I); + return; + } + + assert(I.isUnsigned()); + if ((isa(I.getOperand(0)) || isa(I.getOperand(1)))) { + handleRelationalComparisonExact(I); + return; + } + + handleShadowOr(I); } void visitFCmpInst(FCmpInst &I) { @@ -1231,7 +1529,7 @@ struct MemorySanitizerVisitor : public InstVisitor { const int UnknownModRefBehavior = IK_WritesMemory; #define GET_INTRINSIC_MODREF_BEHAVIOR #define ModRefBehavior IntrinsicKind -#include "llvm/Intrinsics.gen" +#include "llvm/IR/Intrinsics.gen" #undef ModRefBehavior #undef GET_INTRINSIC_MODREF_BEHAVIOR } @@ -1269,16 +1567,25 @@ struct MemorySanitizerVisitor : public InstVisitor { Value *Addr = I.getArgOperand(0); Type *ShadowTy = getShadowTy(&I); - Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB); - // We don't know the pointer alignment (could be unaligned SSE load!). - // Have to assume to worst case. - setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, 1, "_msld")); + if (LoadShadow) { + Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB); + // We don't know the pointer alignment (could be unaligned SSE load!). + // Have to assume to worst case. + setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, 1, "_msld")); + } else { + setShadow(&I, getCleanShadow(&I)); + } + if (ClCheckAccessAddress) insertCheck(Addr, &I); - if (MS.TrackOrigins) - setOrigin(&I, IRB.CreateLoad(getOriginPtr(Addr, IRB))); + if (MS.TrackOrigins) { + if (LoadShadow) + setOrigin(&I, IRB.CreateLoad(getOriginPtr(Addr, IRB))); + else + setOrigin(&I, getCleanOrigin()); + } return true; } @@ -1380,6 +1687,17 @@ struct MemorySanitizerVisitor : public InstVisitor { } } + // Replace call to (*Fn) with a call to (*IndirectCallWrapperFn(Fn)). + void wrapIndirectCall(IRBuilder<> &IRB, CallSite CS) { + Value *Fn = CS.getCalledValue(); + Value *NewFn = IRB.CreateBitCast( + IRB.CreateCall(MS.IndirectCallWrapperFn, + IRB.CreateBitCast(Fn, MS.AnyFunctionPtrTy)), + Fn->getType()); + setShadow(NewFn, getShadow(Fn)); + CS.setCalledFunction(NewFn); + } + void visitCallSite(CallSite CS) { Instruction &I = *CS.getInstruction(); assert((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite"); @@ -1411,11 +1729,17 @@ struct MemorySanitizerVisitor : public InstVisitor { AttrBuilder B; B.addAttribute(Attribute::ReadOnly) .addAttribute(Attribute::ReadNone); - Func->removeAttribute(AttributeSet::FunctionIndex, - Attribute::get(Func->getContext(), B)); + Func->removeAttributes(AttributeSet::FunctionIndex, + AttributeSet::get(Func->getContext(), + AttributeSet::FunctionIndex, + B)); } } IRBuilder<> IRB(&I); + + if (MS.WrapIndirectCalls && !CS.getCalledFunction()) + wrapIndirectCall(IRB, CS); + unsigned ArgOffset = 0; DEBUG(dbgs() << " CallSite: " << I << "\n"); for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); @@ -1451,6 +1775,7 @@ struct MemorySanitizerVisitor : public InstVisitor { if (MS.TrackOrigins) IRB.CreateStore(getOrigin(A), getOriginPtrForArgument(A, IRB, ArgOffset)); + (void)Store; assert(Size != 0 && Store != 0); DEBUG(dbgs() << " Param:" << *Store << "\n"); ArgOffset += DataLayout::RoundUpAlignment(Size, 8); @@ -1458,7 +1783,7 @@ struct MemorySanitizerVisitor : public InstVisitor { DEBUG(dbgs() << " done with call args\n"); FunctionType *FT = - cast(CS.getCalledValue()->getType()-> getContainedType(0)); + cast(CS.getCalledValue()->getType()->getContainedType(0)); if (FT->isVarArg()) { VAHelper->visitCallSite(CS, IRB); } @@ -1497,12 +1822,17 @@ struct MemorySanitizerVisitor : public InstVisitor { void visitReturnInst(ReturnInst &I) { IRBuilder<> IRB(&I); - if (Value *RetVal = I.getReturnValue()) { - // Set the shadow for the RetVal. + Value *RetVal = I.getReturnValue(); + if (!RetVal) return; + Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB); + if (CheckReturnValue) { + insertCheck(RetVal, &I); + Value *Shadow = getCleanShadow(RetVal); + IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment); + } else { Value *Shadow = getShadow(RetVal); - Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB); - DEBUG(dbgs() << "Return: " << *Shadow << "\n" << *ShadowPtr << "\n"); IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment); + // FIXME: make it conditional if ClStoreCleanOrigin==0 if (MS.TrackOrigins) IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB)); } @@ -1520,20 +1850,19 @@ struct MemorySanitizerVisitor : public InstVisitor { void visitAllocaInst(AllocaInst &I) { setShadow(&I, getCleanShadow(&I)); - if (!ClPoisonStack) return; IRBuilder<> IRB(I.getNextNode()); uint64_t Size = MS.TD->getTypeAllocSize(I.getAllocatedType()); - if (ClPoisonStackWithCall) { + if (PoisonStack && ClPoisonStackWithCall) { IRB.CreateCall2(MS.MsanPoisonStackFn, IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), ConstantInt::get(MS.IntptrTy, Size)); } else { Value *ShadowBase = getShadowPtr(&I, Type::getInt8PtrTy(*MS.C), IRB); - IRB.CreateMemSet(ShadowBase, IRB.getInt8(ClPoisonStackPattern), - Size, I.getAlignment()); + Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0); + IRB.CreateMemSet(ShadowBase, PoisonValue, Size, I.getAlignment()); } - if (MS.TrackOrigins) { + if (PoisonStack && MS.TrackOrigins) { setOrigin(&I, getCleanOrigin()); SmallString<2048> StackDescriptionStorage; raw_svector_ostream StackDescription(StackDescriptionStorage); @@ -1546,21 +1875,46 @@ struct MemorySanitizerVisitor : public InstVisitor { Value *Descr = createPrivateNonConstGlobalForString(*F.getParent(), StackDescription.str()); - IRB.CreateCall3(MS.MsanSetAllocaOriginFn, + + IRB.CreateCall4(MS.MsanSetAllocaOrigin4Fn, IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), ConstantInt::get(MS.IntptrTy, Size), - IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())); + IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()), + IRB.CreatePointerCast(&F, MS.IntptrTy)); } } void visitSelectInst(SelectInst& I) { IRBuilder<> IRB(&I); - setShadow(&I, IRB.CreateSelect(I.getCondition(), - getShadow(I.getTrueValue()), getShadow(I.getFalseValue()), - "_msprop")); - if (MS.TrackOrigins) - setOrigin(&I, IRB.CreateSelect(I.getCondition(), + // a = select b, c, d + Value *S = IRB.CreateSelect(I.getCondition(), getShadow(I.getTrueValue()), + getShadow(I.getFalseValue())); + if (I.getType()->isAggregateType()) { + // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do + // an extra "select". This results in much more compact IR. + // Sa = select Sb, poisoned, (select b, Sc, Sd) + S = IRB.CreateSelect(getShadow(I.getCondition()), + getPoisonedShadow(getShadowTy(I.getType())), S, + "_msprop_select_agg"); + } else { + // Sa = (sext Sb) | (select b, Sc, Sd) + S = IRB.CreateOr( + S, IRB.CreateSExt(getShadow(I.getCondition()), S->getType()), + "_msprop_select"); + } + setShadow(&I, S); + if (MS.TrackOrigins) { + // Origins are always i32, so any vector conditions must be flattened. + // FIXME: consider tracking vector origins for app vectors? + Value *Cond = I.getCondition(); + if (Cond->getType()->isVectorTy()) { + Value *ConvertedShadow = convertToShadowTyNoVec(Cond, IRB); + Cond = IRB.CreateICmpNE(ConvertedShadow, + getCleanShadow(ConvertedShadow), "_mso_select"); + } + setOrigin(&I, IRB.CreateSelect(Cond, getOrigin(I.getTrueValue()), getOrigin(I.getFalseValue()))); + } } void visitLandingPadInst(LandingPadInst &I) { @@ -1718,7 +2072,7 @@ struct VarArgAMD64Helper : public VarArgHelper { // Unpoison the whole __va_list_tag. // FIXME: magic ABI constants. IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), - /* size */24, /* alignment */16, false); + /* size */24, /* alignment */8, false); } void visitVACopyInst(VACopyInst &I) { @@ -1729,7 +2083,7 @@ struct VarArgAMD64Helper : public VarArgHelper { // Unpoison the whole __va_list_tag. // FIXME: magic ABI constants. IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), - /* size */ 24, /* alignment */ 16, false); + /* size */24, /* alignment */8, false); } void finalizeInstrumentation() { @@ -1773,16 +2127,35 @@ struct VarArgAMD64Helper : public VarArgHelper { Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr); Value *OverflowArgAreaShadowPtr = MSV.getShadowPtr(OverflowArgAreaPtr, IRB.getInt8Ty(), IRB); - Value *SrcPtr = - getShadowPtrForVAArgument(VAArgTLSCopy, IRB, AMD64FpEndOffset); + Value *SrcPtr = IRB.CreateConstGEP1_32(VAArgTLSCopy, AMD64FpEndOffset); IRB.CreateMemCpy(OverflowArgAreaShadowPtr, SrcPtr, VAArgOverflowSize, 16); } } }; -VarArgHelper* CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, +/// \brief A no-op implementation of VarArgHelper. +struct VarArgNoOpHelper : public VarArgHelper { + VarArgNoOpHelper(Function &F, MemorySanitizer &MS, + MemorySanitizerVisitor &MSV) {} + + void visitCallSite(CallSite &CS, IRBuilder<> &IRB) {} + + void visitVAStartInst(VAStartInst &I) {} + + void visitVACopyInst(VACopyInst &I) {} + + void finalizeInstrumentation() {} +}; + +VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, MemorySanitizerVisitor &Visitor) { - return new VarArgAMD64Helper(Func, Msan, Visitor); + // VarArg handling is only implemented on AMD64. False positives are possible + // on other platforms. + llvm::Triple TargetTriple(Func.getParent()->getTargetTriple()); + if (TargetTriple.getArch() == llvm::Triple::x86_64) + return new VarArgAMD64Helper(Func, Msan, Visitor); + else + return new VarArgNoOpHelper(Func, Msan, Visitor); } } // namespace @@ -1794,8 +2167,9 @@ bool MemorySanitizer::runOnFunction(Function &F) { AttrBuilder B; B.addAttribute(Attribute::ReadOnly) .addAttribute(Attribute::ReadNone); - F.removeAttribute(AttributeSet::FunctionIndex, - Attribute::get(F.getContext(), B)); + F.removeAttributes(AttributeSet::FunctionIndex, + AttributeSet::get(F.getContext(), + AttributeSet::FunctionIndex, B)); return Visitor.runOnFunction(); }