X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=blobdiff_plain;f=lib%2FTransforms%2FInstrumentation%2FAddressSanitizer.cpp;h=8eb82e39b8a689d9906b2a97415fd1b6d389d9cf;hp=6dbcde03cf2db1e040347e062ef9044d9de3e0a1;hb=38e5db618a6629a7273ae04210db2635a68bc4a7;hpb=6140939365cb1d23a7f8ab99b6b30650d41dac42 diff --git a/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/lib/Transforms/Instrumentation/AddressSanitizer.cpp index 6dbcde03cf2..8eb82e39b8a 100644 --- a/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -16,16 +16,22 @@ #include "llvm/Transforms/Instrumentation.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Triple.h" +#include "llvm/Analysis/MemoryBuiltins.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InlineAsm.h" @@ -35,16 +41,20 @@ #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Endian.h" +#include "llvm/Support/SwapByteOrder.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/ASanStackFrameLayout.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/ModuleUtils.h" -#include "llvm/Transforms/Utils/SpecialCaseList.h" +#include "llvm/Transforms/Utils/PromoteMemToReg.h" #include #include #include @@ -58,37 +68,40 @@ static const uint64_t kDefaultShadowOffset32 = 1ULL << 29; static const uint64_t kIOSShadowOffset32 = 1ULL << 30; static const uint64_t kDefaultShadowOffset64 = 1ULL << 44; static const uint64_t kSmallX86_64ShadowOffset = 0x7FFF8000; // < 2G. +static const uint64_t kLinuxKasan_ShadowOffset64 = 0xdffffc0000000000; static const uint64_t kPPC64_ShadowOffset64 = 1ULL << 41; -static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa8000; +static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000; +static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37; +static const uint64_t kAArch64_ShadowOffset64 = 1ULL << 36; static const uint64_t kFreeBSD_ShadowOffset32 = 1ULL << 30; static const uint64_t kFreeBSD_ShadowOffset64 = 1ULL << 46; +static const uint64_t kWindowsShadowOffset32 = 3ULL << 28; -static const size_t kMinStackMallocSize = 1 << 6; // 64B +static const size_t kMinStackMallocSize = 1 << 6; // 64B static const size_t kMaxStackMallocSize = 1 << 16; // 64K static const uintptr_t kCurrentStackFrameMagic = 0x41B58AB3; static const uintptr_t kRetiredStackFrameMagic = 0x45E0360E; static const char *const kAsanModuleCtorName = "asan.module_ctor"; static const char *const kAsanModuleDtorName = "asan.module_dtor"; -static const int kAsanCtorAndDtorPriority = 1; +static const uint64_t kAsanCtorAndDtorPriority = 1; static const char *const kAsanReportErrorTemplate = "__asan_report_"; -static const char *const kAsanReportLoadN = "__asan_report_load_n"; -static const char *const kAsanReportStoreN = "__asan_report_store_n"; static const char *const kAsanRegisterGlobalsName = "__asan_register_globals"; static const char *const kAsanUnregisterGlobalsName = "__asan_unregister_globals"; static const char *const kAsanPoisonGlobalsName = "__asan_before_dynamic_init"; static const char *const kAsanUnpoisonGlobalsName = "__asan_after_dynamic_init"; -static const char *const kAsanInitName = "__asan_init_v3"; -static const char *const kAsanCovModuleInitName = "__sanitizer_cov_module_init"; -static const char *const kAsanCovName = "__sanitizer_cov"; +static const char *const kAsanInitName = "__asan_init"; +static const char *const kAsanVersionCheckName = + "__asan_version_mismatch_check_v6"; static const char *const kAsanPtrCmp = "__sanitizer_ptr_cmp"; static const char *const kAsanPtrSub = "__sanitizer_ptr_sub"; static const char *const kAsanHandleNoReturnName = "__asan_handle_no_return"; -static const int kMaxAsanStackMallocSizeClass = 10; +static const int kMaxAsanStackMallocSizeClass = 10; static const char *const kAsanStackMallocNameTemplate = "__asan_stack_malloc_"; static const char *const kAsanStackFreeNameTemplate = "__asan_stack_free_"; static const char *const kAsanGenPrefix = "__asan_gen_"; +static const char *const kSanCovGenPrefix = "__sancov_gen_"; static const char *const kAsanPoisonStackMemoryName = "__asan_poison_stack_memory"; static const char *const kAsanUnpoisonStackMemoryName = @@ -97,111 +110,129 @@ static const char *const kAsanUnpoisonStackMemoryName = static const char *const kAsanOptionDetectUAR = "__asan_option_detect_stack_use_after_return"; -#ifndef NDEBUG -static const int kAsanStackAfterReturnMagic = 0xf5; -#endif +static const char *const kAsanAllocaPoison = "__asan_alloca_poison"; +static const char *const kAsanAllocasUnpoison = "__asan_allocas_unpoison"; // Accesses sizes are powers of two: 1, 2, 4, 8, 16. static const size_t kNumberOfAccessSizes = 5; +static const unsigned kAllocaRzSize = 32; + // Command-line flags. +static cl::opt ClEnableKasan( + "asan-kernel", cl::desc("Enable KernelAddressSanitizer instrumentation"), + cl::Hidden, cl::init(false)); +static cl::opt ClRecover( + "asan-recover", + cl::desc("Enable recovery mode (continue-after-error)."), + cl::Hidden, cl::init(false)); // This flag may need to be replaced with -f[no-]asan-reads. static cl::opt ClInstrumentReads("asan-instrument-reads", - cl::desc("instrument read instructions"), cl::Hidden, cl::init(true)); -static cl::opt ClInstrumentWrites("asan-instrument-writes", - cl::desc("instrument write instructions"), cl::Hidden, cl::init(true)); -static cl::opt ClInstrumentAtomics("asan-instrument-atomics", - cl::desc("instrument atomic instructions (rmw, cmpxchg)"), - cl::Hidden, cl::init(true)); -static cl::opt ClAlwaysSlowPath("asan-always-slow-path", - cl::desc("use instrumentation with slow path for all accesses"), - cl::Hidden, cl::init(false)); + cl::desc("instrument read instructions"), + cl::Hidden, cl::init(true)); +static cl::opt ClInstrumentWrites( + "asan-instrument-writes", cl::desc("instrument write instructions"), + cl::Hidden, cl::init(true)); +static cl::opt ClInstrumentAtomics( + "asan-instrument-atomics", + cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, + cl::init(true)); +static cl::opt ClAlwaysSlowPath( + "asan-always-slow-path", + cl::desc("use instrumentation with slow path for all accesses"), cl::Hidden, + cl::init(false)); // This flag limits the number of instructions to be instrumented // in any given BB. Normally, this should be set to unlimited (INT_MAX), // but due to http://llvm.org/bugs/show_bug.cgi?id=12652 we temporary // set it to 10000. -static cl::opt ClMaxInsnsToInstrumentPerBB("asan-max-ins-per-bb", - cl::init(10000), - cl::desc("maximal number of instructions to instrument in any given BB"), - cl::Hidden); +static cl::opt ClMaxInsnsToInstrumentPerBB( + "asan-max-ins-per-bb", cl::init(10000), + cl::desc("maximal number of instructions to instrument in any given BB"), + cl::Hidden); // This flag may need to be replaced with -f[no]asan-stack. -static cl::opt ClStack("asan-stack", - cl::desc("Handle stack memory"), cl::Hidden, cl::init(true)); +static cl::opt ClStack("asan-stack", cl::desc("Handle stack memory"), + cl::Hidden, cl::init(true)); static cl::opt ClUseAfterReturn("asan-use-after-return", - cl::desc("Check return-after-free"), cl::Hidden, cl::init(true)); + cl::desc("Check return-after-free"), + cl::Hidden, cl::init(true)); // This flag may need to be replaced with -f[no]asan-globals. static cl::opt ClGlobals("asan-globals", - cl::desc("Handle global objects"), cl::Hidden, cl::init(true)); -static cl::opt ClCoverage("asan-coverage", - cl::desc("ASan coverage. 0: none, 1: entry block, 2: all blocks"), - cl::Hidden, cl::init(false)); -static cl::opt ClCoverageBlockThreshold("asan-coverage-block-threshold", - cl::desc("Add coverage instrumentation only to the entry block if there " - "are more than this number of blocks."), - cl::Hidden, cl::init(1500)); + cl::desc("Handle global objects"), cl::Hidden, + cl::init(true)); static cl::opt ClInitializers("asan-initialization-order", - cl::desc("Handle C++ initializer order"), cl::Hidden, cl::init(true)); -static cl::opt ClInvalidPointerPairs("asan-detect-invalid-pointer-pair", - cl::desc("Instrument <, <=, >, >=, - with pointer operands"), - cl::Hidden, cl::init(false)); -static cl::opt ClRealignStack("asan-realign-stack", - cl::desc("Realign stack to the value of this flag (power of two)"), - cl::Hidden, cl::init(32)); -static cl::opt ClBlacklistFile("asan-blacklist", - cl::desc("File containing the list of objects to ignore " - "during instrumentation"), cl::Hidden); + cl::desc("Handle C++ initializer order"), + cl::Hidden, cl::init(true)); +static cl::opt ClInvalidPointerPairs( + "asan-detect-invalid-pointer-pair", + cl::desc("Instrument <, <=, >, >=, - with pointer operands"), cl::Hidden, + cl::init(false)); +static cl::opt ClRealignStack( + "asan-realign-stack", + cl::desc("Realign stack to the value of this flag (power of two)"), + cl::Hidden, cl::init(32)); static cl::opt ClInstrumentationWithCallsThreshold( "asan-instrumentation-with-call-threshold", - cl::desc("If the function being instrumented contains more than " - "this number of memory accesses, use callbacks instead of " - "inline checks (-1 means never use callbacks)."), - cl::Hidden, cl::init(7000)); + cl::desc( + "If the function being instrumented contains more than " + "this number of memory accesses, use callbacks instead of " + "inline checks (-1 means never use callbacks)."), + cl::Hidden, cl::init(7000)); static cl::opt ClMemoryAccessCallbackPrefix( - "asan-memory-access-callback-prefix", - cl::desc("Prefix for memory access callbacks"), cl::Hidden, - cl::init("__asan_")); - -// This is an experimental feature that will allow to choose between -// instrumented and non-instrumented code at link-time. -// If this option is on, just before instrumenting a function we create its -// clone; if the function is not changed by asan the clone is deleted. -// If we end up with a clone, we put the instrumented function into a section -// called "ASAN" and the uninstrumented function into a section called "NOASAN". -// -// This is still a prototype, we need to figure out a way to keep two copies of -// a function so that the linker can easily choose one of them. -static cl::opt ClKeepUninstrumented("asan-keep-uninstrumented-functions", - cl::desc("Keep uninstrumented copies of functions"), - cl::Hidden, cl::init(false)); + "asan-memory-access-callback-prefix", + cl::desc("Prefix for memory access callbacks"), cl::Hidden, + cl::init("__asan_")); +static cl::opt ClInstrumentAllocas("asan-instrument-allocas", + cl::desc("instrument dynamic allocas"), + cl::Hidden, cl::init(true)); +static cl::opt ClSkipPromotableAllocas( + "asan-skip-promotable-allocas", + cl::desc("Do not instrument promotable allocas"), cl::Hidden, + cl::init(true)); // These flags allow to change the shadow mapping. // The shadow mapping looks like // Shadow = (Mem >> scale) + (1 << offset_log) static cl::opt ClMappingScale("asan-mapping-scale", - cl::desc("scale of asan shadow mapping"), cl::Hidden, cl::init(0)); + cl::desc("scale of asan shadow mapping"), + cl::Hidden, cl::init(0)); // Optimization flags. Not user visible, used mostly for testing // and benchmarking the tool. -static cl::opt ClOpt("asan-opt", - cl::desc("Optimize instrumentation"), cl::Hidden, cl::init(true)); -static cl::opt ClOptSameTemp("asan-opt-same-temp", - cl::desc("Instrument the same temp just once"), cl::Hidden, - cl::init(true)); +static cl::opt ClOpt("asan-opt", cl::desc("Optimize instrumentation"), + cl::Hidden, cl::init(true)); +static cl::opt ClOptSameTemp( + "asan-opt-same-temp", cl::desc("Instrument the same temp just once"), + cl::Hidden, cl::init(true)); static cl::opt ClOptGlobals("asan-opt-globals", - cl::desc("Don't instrument scalar globals"), cl::Hidden, cl::init(true)); - -static cl::opt ClCheckLifetime("asan-check-lifetime", - cl::desc("Use llvm.lifetime intrinsics to insert extra checks"), - cl::Hidden, cl::init(false)); + cl::desc("Don't instrument scalar globals"), + cl::Hidden, cl::init(true)); +static cl::opt ClOptStack( + "asan-opt-stack", cl::desc("Don't instrument scalar stack variables"), + cl::Hidden, cl::init(false)); + +static cl::opt ClCheckLifetime( + "asan-check-lifetime", + cl::desc("Use llvm.lifetime intrinsics to insert extra checks"), cl::Hidden, + cl::init(false)); + +static cl::opt ClDynamicAllocaStack( + "asan-stack-dynamic-alloca", + cl::desc("Use dynamic alloca to represent stack variables"), cl::Hidden, + cl::init(true)); + +static cl::opt ClForceExperiment( + "asan-force-experiment", + cl::desc("Force optimization experiment (for testing)"), cl::Hidden, + cl::init(0)); // Debug flags. static cl::opt ClDebug("asan-debug", cl::desc("debug"), cl::Hidden, cl::init(0)); static cl::opt ClDebugStack("asan-debug-stack", cl::desc("debug stack"), cl::Hidden, cl::init(0)); -static cl::opt ClDebugFunc("asan-debug-func", - cl::Hidden, cl::desc("Debug func")); +static cl::opt ClDebugFunc("asan-debug-func", cl::Hidden, + cl::desc("Debug func")); static cl::opt ClDebugMin("asan-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1)); static cl::opt ClDebugMax("asan-debug-max", cl::desc("Debug man inst"), @@ -209,34 +240,82 @@ static cl::opt ClDebugMax("asan-debug-max", cl::desc("Debug man inst"), STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); -STATISTIC(NumOptimizedAccessesToGlobalArray, - "Number of optimized accesses to global arrays"); STATISTIC(NumOptimizedAccessesToGlobalVar, "Number of optimized accesses to global vars"); +STATISTIC(NumOptimizedAccessesToStackVar, + "Number of optimized accesses to stack vars"); namespace { -/// A set of dynamically initialized globals extracted from metadata. -class SetOfDynamicallyInitializedGlobals { +/// Frontend-provided metadata for source location. +struct LocationMetadata { + StringRef Filename; + int LineNo; + int ColumnNo; + + LocationMetadata() : Filename(), LineNo(0), ColumnNo(0) {} + + bool empty() const { return Filename.empty(); } + + void parse(MDNode *MDN) { + assert(MDN->getNumOperands() == 3); + MDString *DIFilename = cast(MDN->getOperand(0)); + Filename = DIFilename->getString(); + LineNo = + mdconst::extract(MDN->getOperand(1))->getLimitedValue(); + ColumnNo = + mdconst::extract(MDN->getOperand(2))->getLimitedValue(); + } +}; + +/// Frontend-provided metadata for global variables. +class GlobalsMetadata { public: - void Init(Module& M) { - // Clang generates metadata identifying all dynamically initialized globals. - NamedMDNode *DynamicGlobals = - M.getNamedMetadata("llvm.asan.dynamically_initialized_globals"); - if (!DynamicGlobals) - return; - for (const auto MDN : DynamicGlobals->operands()) { - assert(MDN->getNumOperands() == 1); - Value *VG = MDN->getOperand(0); - // The optimizer may optimize away a global entirely, in which case we - // cannot instrument access to it. - if (!VG) - continue; - DynInitGlobals.insert(cast(VG)); + struct Entry { + Entry() : SourceLoc(), Name(), IsDynInit(false), IsBlacklisted(false) {} + LocationMetadata SourceLoc; + StringRef Name; + bool IsDynInit; + bool IsBlacklisted; + }; + + GlobalsMetadata() : inited_(false) {} + + void init(Module &M) { + assert(!inited_); + inited_ = true; + NamedMDNode *Globals = M.getNamedMetadata("llvm.asan.globals"); + if (!Globals) return; + for (auto MDN : Globals->operands()) { + // Metadata node contains the global and the fields of "Entry". + assert(MDN->getNumOperands() == 5); + auto *GV = mdconst::extract_or_null(MDN->getOperand(0)); + // The optimizer may optimize away a global entirely. + if (!GV) continue; + // We can already have an entry for GV if it was merged with another + // global. + Entry &E = Entries[GV]; + if (auto *Loc = cast_or_null(MDN->getOperand(1))) + E.SourceLoc.parse(Loc); + if (auto *Name = cast_or_null(MDN->getOperand(2))) + E.Name = Name->getString(); + ConstantInt *IsDynInit = + mdconst::extract(MDN->getOperand(3)); + E.IsDynInit |= IsDynInit->isOne(); + ConstantInt *IsBlacklisted = + mdconst::extract(MDN->getOperand(4)); + E.IsBlacklisted |= IsBlacklisted->isOne(); } } - bool Contains(GlobalVariable *G) { return DynInitGlobals.count(G) != 0; } + + /// Returns metadata entry for a given global. + Entry get(GlobalVariable *G) const { + auto Pos = Entries.find(G); + return (Pos != Entries.end()) ? Pos->second : Entry(); + } + private: - SmallSet DynInitGlobals; + bool inited_; + DenseMap Entries; }; /// This struct defines the shadow mapping using the rule: @@ -247,21 +326,27 @@ struct ShadowMapping { bool OrShadowOffset; }; -static ShadowMapping getShadowMapping(const Module &M, int LongSize) { - llvm::Triple TargetTriple(M.getTargetTriple()); - bool IsAndroid = TargetTriple.getEnvironment() == llvm::Triple::Android; - bool IsIOS = TargetTriple.getOS() == llvm::Triple::IOS; - bool IsFreeBSD = TargetTriple.getOS() == llvm::Triple::FreeBSD; - bool IsLinux = TargetTriple.getOS() == llvm::Triple::Linux; +static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize, + bool IsKasan) { + bool IsAndroid = TargetTriple.isAndroid(); + bool IsIOS = TargetTriple.isiOS(); + bool IsFreeBSD = TargetTriple.isOSFreeBSD(); + bool IsLinux = TargetTriple.isOSLinux(); bool IsPPC64 = TargetTriple.getArch() == llvm::Triple::ppc64 || TargetTriple.getArch() == llvm::Triple::ppc64le; bool IsX86_64 = TargetTriple.getArch() == llvm::Triple::x86_64; bool IsMIPS32 = TargetTriple.getArch() == llvm::Triple::mips || TargetTriple.getArch() == llvm::Triple::mipsel; + bool IsMIPS64 = TargetTriple.getArch() == llvm::Triple::mips64 || + TargetTriple.getArch() == llvm::Triple::mips64el; + bool IsAArch64 = TargetTriple.getArch() == llvm::Triple::aarch64; + bool IsWindows = TargetTriple.isOSWindows(); ShadowMapping Mapping; if (LongSize == 32) { + // Android is always PIE, which means that the beginning of the address + // space is always available. if (IsAndroid) Mapping.Offset = 0; else if (IsMIPS32) @@ -270,6 +355,8 @@ static ShadowMapping getShadowMapping(const Module &M, int LongSize) { Mapping.Offset = kFreeBSD_ShadowOffset32; else if (IsIOS) Mapping.Offset = kIOSShadowOffset32; + else if (IsWindows) + Mapping.Offset = kWindowsShadowOffset32; else Mapping.Offset = kDefaultShadowOffset32; } else { // LongSize == 64 @@ -277,8 +364,15 @@ static ShadowMapping getShadowMapping(const Module &M, int LongSize) { Mapping.Offset = kPPC64_ShadowOffset64; else if (IsFreeBSD) Mapping.Offset = kFreeBSD_ShadowOffset64; - else if (IsLinux && IsX86_64) - Mapping.Offset = kSmallX86_64ShadowOffset; + else if (IsLinux && IsX86_64) { + if (IsKasan) + Mapping.Offset = kLinuxKasan_ShadowOffset64; + else + Mapping.Offset = kSmallX86_64ShadowOffset; + } else if (IsMIPS64) + Mapping.Offset = kMIPS64_ShadowOffset64; + else if (IsAArch64) + Mapping.Offset = kAArch64_ShadowOffset64; else Mapping.Offset = kDefaultShadowOffset64; } @@ -291,7 +385,8 @@ static ShadowMapping getShadowMapping(const Module &M, int LongSize) { // OR-ing shadow offset if more efficient (at least on x86) if the offset // is a power of two, but on ppc64 we have to use add since the shadow // offset is not necessary 1/8-th of the address space. - Mapping.OrShadowOffset = !IsPPC64 && !(Mapping.Offset & (Mapping.Offset - 1)); + Mapping.OrShadowOffset = !IsAArch64 && !IsPPC64 + && !(Mapping.Offset & (Mapping.Offset - 1)); return Mapping; } @@ -304,68 +399,114 @@ static size_t RedzoneSizeForScale(int MappingScale) { /// AddressSanitizer: instrument the code in module to find memory bugs. struct AddressSanitizer : public FunctionPass { - AddressSanitizer() : FunctionPass(ID) {} + explicit AddressSanitizer(bool CompileKernel = false, bool Recover = false) + : FunctionPass(ID), CompileKernel(CompileKernel || ClEnableKasan), + Recover(Recover || ClRecover) { + initializeAddressSanitizerPass(*PassRegistry::getPassRegistry()); + } const char *getPassName() const override { return "AddressSanitizerFunctionPass"; } - void instrumentMop(Instruction *I, bool UseCalls); + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + } + uint64_t getAllocaSizeInBytes(AllocaInst *AI) const { + Type *Ty = AI->getAllocatedType(); + uint64_t SizeInBytes = + AI->getModule()->getDataLayout().getTypeAllocSize(Ty); + return SizeInBytes; + } + /// Check if we want (and can) handle this alloca. + bool isInterestingAlloca(AllocaInst &AI); + + // Check if we have dynamic alloca. + bool isDynamicAlloca(AllocaInst &AI) const { + return AI.isArrayAllocation() || !AI.isStaticAlloca(); + } + + /// If it is an interesting memory access, return the PointerOperand + /// and set IsWrite/Alignment. Otherwise return nullptr. + Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite, + uint64_t *TypeSize, unsigned *Alignment); + void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis, Instruction *I, + bool UseCalls, const DataLayout &DL); void instrumentPointerComparisonOrSubtraction(Instruction *I); void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore, Value *Addr, uint32_t TypeSize, bool IsWrite, - Value *SizeArgument, bool UseCalls); + Value *SizeArgument, bool UseCalls, uint32_t Exp); + void instrumentUnusualSizeOrAlignment(Instruction *I, Value *Addr, + uint32_t TypeSize, bool IsWrite, + Value *SizeArgument, bool UseCalls, + uint32_t Exp); Value *createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong, Value *ShadowValue, uint32_t TypeSize); Instruction *generateCrashCode(Instruction *InsertBefore, Value *Addr, bool IsWrite, size_t AccessSizeIndex, - Value *SizeArgument); + Value *SizeArgument, uint32_t Exp); void instrumentMemIntrinsic(MemIntrinsic *MI); Value *memToShadow(Value *Shadow, IRBuilder<> &IRB); bool runOnFunction(Function &F) override; bool maybeInsertAsanInitAtFunctionEntry(Function &F); + void markEscapedLocalAllocas(Function &F); bool doInitialization(Module &M) override; static char ID; // Pass identification, replacement for typeid + DominatorTree &getDominatorTree() const { return *DT; } + private: void initializeCallbacks(Module &M); bool LooksLikeCodeInBug11395(Instruction *I); bool GlobalIsLinkerInitialized(GlobalVariable *G); - bool InjectCoverage(Function &F, const ArrayRef AllBlocks); - void InjectCoverageAtBlock(Function &F, BasicBlock &BB); + bool isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis, Value *Addr, + uint64_t TypeSize) const; + + /// Helper to cleanup per-function state. + struct FunctionStateRAII { + AddressSanitizer *Pass; + FunctionStateRAII(AddressSanitizer *Pass) : Pass(Pass) { + assert(Pass->ProcessedAllocas.empty() && + "last pass forgot to clear cache"); + } + ~FunctionStateRAII() { Pass->ProcessedAllocas.clear(); } + }; LLVMContext *C; - const DataLayout *DL; + Triple TargetTriple; int LongSize; + bool CompileKernel; + bool Recover; Type *IntptrTy; ShadowMapping Mapping; - Function *AsanCtorFunction; - Function *AsanInitFunction; + DominatorTree *DT; + Function *AsanCtorFunction = nullptr; + Function *AsanInitFunction = nullptr; Function *AsanHandleNoReturnFunc; - Function *AsanCovFunction; Function *AsanPtrCmpFunction, *AsanPtrSubFunction; - // This array is indexed by AccessIsWrite and log2(AccessSize). - Function *AsanErrorCallback[2][kNumberOfAccessSizes]; - Function *AsanMemoryAccessCallback[2][kNumberOfAccessSizes]; - // This array is indexed by AccessIsWrite. - Function *AsanErrorCallbackSized[2], - *AsanMemoryAccessCallbackSized[2]; + // This array is indexed by AccessIsWrite, Experiment and log2(AccessSize). + Function *AsanErrorCallback[2][2][kNumberOfAccessSizes]; + Function *AsanMemoryAccessCallback[2][2][kNumberOfAccessSizes]; + // This array is indexed by AccessIsWrite and Experiment. + Function *AsanErrorCallbackSized[2][2]; + Function *AsanMemoryAccessCallbackSized[2][2]; Function *AsanMemmove, *AsanMemcpy, *AsanMemset; InlineAsm *EmptyAsm; - SetOfDynamicallyInitializedGlobals DynamicallyInitializedGlobals; + GlobalsMetadata GlobalsMD; + DenseMap ProcessedAllocas; friend struct FunctionStackPoisoner; }; class AddressSanitizerModule : public ModulePass { public: - AddressSanitizerModule(StringRef BlacklistFile = StringRef()) - : ModulePass(ID), BlacklistFile(BlacklistFile.empty() ? ClBlacklistFile - : BlacklistFile) {} + explicit AddressSanitizerModule(bool CompileKernel = false, + bool Recover = false) + : ModulePass(ID), CompileKernel(CompileKernel || ClEnableKasan), + Recover(Recover || ClRecover) {} bool runOnModule(Module &M) override; static char ID; // Pass identification, replacement for typeid - const char *getPassName() const override { - return "AddressSanitizerModule"; - } + const char *getPassName() const override { return "AddressSanitizerModule"; } private: void initializeCallbacks(Module &M); @@ -378,19 +519,17 @@ class AddressSanitizerModule : public ModulePass { return RedzoneSizeForScale(Mapping.Scale); } - SmallString<64> BlacklistFile; - - std::unique_ptr BL; - SetOfDynamicallyInitializedGlobals DynamicallyInitializedGlobals; + GlobalsMetadata GlobalsMD; + bool CompileKernel; + bool Recover; Type *IntptrTy; LLVMContext *C; - const DataLayout *DL; + Triple TargetTriple; ShadowMapping Mapping; Function *AsanPoisonGlobals; Function *AsanUnpoisonGlobals; Function *AsanRegisterGlobals; Function *AsanUnregisterGlobals; - Function *AsanCovModuleInit; }; // Stack poisoning does not play well with exception handling. @@ -411,13 +550,15 @@ struct FunctionStackPoisoner : public InstVisitor { Type *IntptrPtrTy; ShadowMapping Mapping; - SmallVector AllocaVec; - SmallVector RetVec; + SmallVector AllocaVec; + SmallSetVector NonInstrumentedStaticAllocaVec; + SmallVector RetVec; unsigned StackAlignment; Function *AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1], - *AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1]; + *AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1]; Function *AsanPoisonStackMemoryFunc, *AsanUnpoisonStackMemoryFunc; + Function *AsanAllocaPoisonFunc, *AsanAllocasUnpoisonFunc; // Stores a place and arguments of poisoning/unpoisoning call for alloca. struct AllocaPoisonCall { @@ -428,23 +569,36 @@ struct FunctionStackPoisoner : public InstVisitor { }; SmallVector AllocaPoisonCallVec; + SmallVector DynamicAllocaVec; + SmallVector StackRestoreVec; + AllocaInst *DynamicAllocaLayout = nullptr; + IntrinsicInst *LocalEscapeCall = nullptr; + // Maps Value to an AllocaInst from which the Value is originated. - typedef DenseMap AllocaForValueMapTy; + typedef DenseMap AllocaForValueMapTy; AllocaForValueMapTy AllocaForValue; + bool HasNonEmptyInlineAsm = false; + bool HasReturnsTwiceCall = false; + std::unique_ptr EmptyInlineAsm; + FunctionStackPoisoner(Function &F, AddressSanitizer &ASan) - : F(F), ASan(ASan), DIB(*F.getParent()), C(ASan.C), - IntptrTy(ASan.IntptrTy), IntptrPtrTy(PointerType::get(IntptrTy, 0)), + : F(F), + ASan(ASan), + DIB(*F.getParent(), /*AllowUnresolved*/ false), + C(ASan.C), + IntptrTy(ASan.IntptrTy), + IntptrPtrTy(PointerType::get(IntptrTy, 0)), Mapping(ASan.Mapping), - StackAlignment(1 << Mapping.Scale) {} + StackAlignment(1 << Mapping.Scale), + EmptyInlineAsm(CallInst::Create(ASan.EmptyAsm)) {} bool runOnFunction() { if (!ClStack) return false; // Collect alloca, ret, lifetime instructions etc. - for (BasicBlock *BB : depth_first(&F.getEntryBlock())) - visit(*BB); + for (BasicBlock *BB : depth_first(&F.getEntryBlock())) visit(*BB); - if (AllocaVec.empty()) return false; + if (AllocaVec.empty() && DynamicAllocaVec.empty()) return false; initializeCallbacks(*F.getParent()); @@ -456,32 +610,69 @@ struct FunctionStackPoisoner : public InstVisitor { return true; } - // Finds all static Alloca instructions and puts + // Finds all Alloca instructions and puts // poisoned red zones around all of them. // Then unpoison everything back before the function returns. void poisonStack(); + void createDynamicAllocasInitStorage(); + // ----------------------- Visitors. /// \brief Collect all Ret instructions. - void visitReturnInst(ReturnInst &RI) { - RetVec.push_back(&RI); + void visitReturnInst(ReturnInst &RI) { RetVec.push_back(&RI); } + + void unpoisonDynamicAllocasBeforeInst(Instruction *InstBefore, + Value *SavedStack) { + IRBuilder<> IRB(InstBefore); + IRB.CreateCall(AsanAllocasUnpoisonFunc, + {IRB.CreateLoad(DynamicAllocaLayout), + IRB.CreatePtrToInt(SavedStack, IntptrTy)}); } + // Unpoison dynamic allocas redzones. + void unpoisonDynamicAllocas() { + for (auto &Ret : RetVec) + unpoisonDynamicAllocasBeforeInst(Ret, DynamicAllocaLayout); + + for (auto &StackRestoreInst : StackRestoreVec) + unpoisonDynamicAllocasBeforeInst(StackRestoreInst, + StackRestoreInst->getOperand(0)); + } + + // Deploy and poison redzones around dynamic alloca call. To do this, we + // should replace this call with another one with changed parameters and + // replace all its uses with new address, so + // addr = alloca type, old_size, align + // is replaced by + // new_size = (old_size + additional_size) * sizeof(type) + // tmp = alloca i8, new_size, max(align, 32) + // addr = tmp + 32 (first 32 bytes are for the left redzone). + // Additional_size is added to make new memory allocation contain not only + // requested memory, but also left, partial and right redzones. + void handleDynamicAllocaCall(AllocaInst *AI); + /// \brief Collect Alloca instructions we want (and can) handle. void visitAllocaInst(AllocaInst &AI) { - if (!isInterestingAlloca(AI)) return; + if (!ASan.isInterestingAlloca(AI)) { + if (AI.isStaticAlloca()) NonInstrumentedStaticAllocaVec.insert(&AI); + return; + } StackAlignment = std::max(StackAlignment, AI.getAlignment()); - AllocaVec.push_back(&AI); + if (ASan.isDynamicAlloca(AI)) + DynamicAllocaVec.push_back(&AI); + else + AllocaVec.push_back(&AI); } /// \brief Collect lifetime intrinsic calls to check for use-after-scope /// errors. void visitIntrinsicInst(IntrinsicInst &II) { - if (!ClCheckLifetime) return; Intrinsic::ID ID = II.getIntrinsicID(); - if (ID != Intrinsic::lifetime_start && - ID != Intrinsic::lifetime_end) + if (ID == Intrinsic::stackrestore) StackRestoreVec.push_back(&II); + if (ID == Intrinsic::localescape) LocalEscapeCall = &II; + if (!ClCheckLifetime) return; + if (ID != Intrinsic::lifetime_start && ID != Intrinsic::lifetime_end) return; // Found lifetime intrinsic, add ASan instrumentation if necessary. ConstantInt *Size = dyn_cast(II.getArgOperand(0)); @@ -501,48 +692,68 @@ struct FunctionStackPoisoner : public InstVisitor { AllocaPoisonCallVec.push_back(APC); } + void visitCallSite(CallSite CS) { + Instruction *I = CS.getInstruction(); + if (CallInst *CI = dyn_cast(I)) { + HasNonEmptyInlineAsm |= + CI->isInlineAsm() && !CI->isIdenticalTo(EmptyInlineAsm.get()); + HasReturnsTwiceCall |= CI->canReturnTwice(); + } + } + // ---------------------- Helpers. void initializeCallbacks(Module &M); - // Check if we want (and can) handle this alloca. - bool isInterestingAlloca(AllocaInst &AI) const { - return (!AI.isArrayAllocation() && AI.isStaticAlloca() && - AI.getAllocatedType()->isSized() && - // alloca() may be called with 0 size, ignore it. - getAllocaSizeInBytes(&AI) > 0); + bool doesDominateAllExits(const Instruction *I) const { + for (auto Ret : RetVec) { + if (!ASan.getDominatorTree().dominates(I, Ret)) return false; + } + return true; } - uint64_t getAllocaSizeInBytes(AllocaInst *AI) const { - Type *Ty = AI->getAllocatedType(); - uint64_t SizeInBytes = ASan.DL->getTypeAllocSize(Ty); - return SizeInBytes; - } /// Finds alloca where the value comes from. AllocaInst *findAllocaForValue(Value *V); - void poisonRedZones(const ArrayRef ShadowBytes, IRBuilder<> &IRB, + void poisonRedZones(ArrayRef ShadowBytes, IRBuilder<> &IRB, Value *ShadowBase, bool DoPoison); void poisonAlloca(Value *V, uint64_t Size, IRBuilder<> &IRB, bool DoPoison); void SetShadowToStackAfterReturnInlined(IRBuilder<> &IRB, Value *ShadowBase, int Size); + Value *createAllocaForLayout(IRBuilder<> &IRB, const ASanStackFrameLayout &L, + bool Dynamic); + PHINode *createPHI(IRBuilder<> &IRB, Value *Cond, Value *ValueIfTrue, + Instruction *ThenTerm, Value *ValueIfFalse); }; -} // namespace +} // anonymous namespace char AddressSanitizer::ID = 0; -INITIALIZE_PASS(AddressSanitizer, "asan", - "AddressSanitizer: detects use-after-free and out-of-bounds bugs.", - false, false) -FunctionPass *llvm::createAddressSanitizerFunctionPass() { - return new AddressSanitizer(); +INITIALIZE_PASS_BEGIN( + AddressSanitizer, "asan", + "AddressSanitizer: detects use-after-free and out-of-bounds bugs.", false, + false) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) +INITIALIZE_PASS_END( + AddressSanitizer, "asan", + "AddressSanitizer: detects use-after-free and out-of-bounds bugs.", false, + false) +FunctionPass *llvm::createAddressSanitizerFunctionPass(bool CompileKernel, + bool Recover) { + assert(!CompileKernel || Recover); + return new AddressSanitizer(CompileKernel, Recover); } char AddressSanitizerModule::ID = 0; -INITIALIZE_PASS(AddressSanitizerModule, "asan-module", +INITIALIZE_PASS( + AddressSanitizerModule, "asan-module", "AddressSanitizer: detects use-after-free and out-of-bounds bugs." - "ModulePass", false, false) -ModulePass *llvm::createAddressSanitizerModulePass(StringRef BlacklistFile) { - return new AddressSanitizerModule(BlacklistFile); + "ModulePass", + false, false) +ModulePass *llvm::createAddressSanitizerModulePass(bool CompileKernel, + bool Recover) { + assert(!CompileKernel || Recover); + return new AddressSanitizerModule(CompileKernel, Recover); } static size_t TypeSizeToSizeIndex(uint32_t TypeSize) { @@ -552,29 +763,44 @@ static size_t TypeSizeToSizeIndex(uint32_t TypeSize) { } // \brief Create a constant for Str so that we can pass it to the run-time lib. -static GlobalVariable *createPrivateGlobalForString( - Module &M, StringRef Str, bool AllowMerging) { +static GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str, + bool AllowMerging) { Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str); // We use private linkage for module-local strings. If they can be merged // with another one, we set the unnamed_addr attribute. GlobalVariable *GV = new GlobalVariable(M, StrConst->getType(), true, GlobalValue::PrivateLinkage, StrConst, kAsanGenPrefix); - if (AllowMerging) - GV->setUnnamedAddr(true); + if (AllowMerging) GV->setUnnamedAddr(true); GV->setAlignment(1); // Strings may not be merged w/o setting align 1. return GV; } +/// \brief Create a global describing a source location. +static GlobalVariable *createPrivateGlobalForSourceLoc(Module &M, + LocationMetadata MD) { + Constant *LocData[] = { + createPrivateGlobalForString(M, MD.Filename, true), + ConstantInt::get(Type::getInt32Ty(M.getContext()), MD.LineNo), + ConstantInt::get(Type::getInt32Ty(M.getContext()), MD.ColumnNo), + }; + auto LocStruct = ConstantStruct::getAnon(LocData); + auto GV = new GlobalVariable(M, LocStruct->getType(), true, + GlobalValue::PrivateLinkage, LocStruct, + kAsanGenPrefix); + GV->setUnnamedAddr(true); + return GV; +} + static bool GlobalWasGeneratedByAsan(GlobalVariable *G) { - return G->getName().find(kAsanGenPrefix) == 0; + return G->getName().find(kAsanGenPrefix) == 0 || + G->getName().find(kSanCovGenPrefix) == 0; } Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) { // Shadow >> scale Shadow = IRB.CreateLShr(Shadow, Mapping.Scale); - if (Mapping.Offset == 0) - return Shadow; + if (Mapping.Offset == 0) return Shadow; // (Shadow >> scale) | offset if (Mapping.OrShadowOffset) return IRB.CreateOr(Shadow, ConstantInt::get(IntptrTy, Mapping.Offset)); @@ -586,50 +812,88 @@ Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) { void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) { IRBuilder<> IRB(MI); if (isa(MI)) { - IRB.CreateCall3( + IRB.CreateCall( isa(MI) ? AsanMemmove : AsanMemcpy, - IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()), - IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()), - IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)); + {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()), + IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()), + IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)}); } else if (isa(MI)) { - IRB.CreateCall3( + IRB.CreateCall( AsanMemset, - IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()), - IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false), - IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)); + {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()), + IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false), + IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)}); } MI->eraseFromParent(); } -// If I is an interesting memory access, return the PointerOperand -// and set IsWrite/Alignment. Otherwise return NULL. -static Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite, - unsigned *Alignment) { +/// Check if we want (and can) handle this alloca. +bool AddressSanitizer::isInterestingAlloca(AllocaInst &AI) { + auto PreviouslySeenAllocaInfo = ProcessedAllocas.find(&AI); + + if (PreviouslySeenAllocaInfo != ProcessedAllocas.end()) + return PreviouslySeenAllocaInfo->getSecond(); + + bool IsInteresting = + (AI.getAllocatedType()->isSized() && + // alloca() may be called with 0 size, ignore it. + getAllocaSizeInBytes(&AI) > 0 && + // We are only interested in allocas not promotable to registers. + // Promotable allocas are common under -O0. + (!ClSkipPromotableAllocas || !isAllocaPromotable(&AI)) && + // inalloca allocas are not treated as static, and we don't want + // dynamic alloca instrumentation for them as well. + !AI.isUsedWithInAlloca()); + + ProcessedAllocas[&AI] = IsInteresting; + return IsInteresting; +} + +/// If I is an interesting memory access, return the PointerOperand +/// and set IsWrite/Alignment. Otherwise return nullptr. +Value *AddressSanitizer::isInterestingMemoryAccess(Instruction *I, + bool *IsWrite, + uint64_t *TypeSize, + unsigned *Alignment) { + // Skip memory accesses inserted by another instrumentation. + if (I->getMetadata("nosanitize")) return nullptr; + + Value *PtrOperand = nullptr; + const DataLayout &DL = I->getModule()->getDataLayout(); if (LoadInst *LI = dyn_cast(I)) { if (!ClInstrumentReads) return nullptr; *IsWrite = false; + *TypeSize = DL.getTypeStoreSizeInBits(LI->getType()); *Alignment = LI->getAlignment(); - return LI->getPointerOperand(); - } - if (StoreInst *SI = dyn_cast(I)) { + PtrOperand = LI->getPointerOperand(); + } else if (StoreInst *SI = dyn_cast(I)) { if (!ClInstrumentWrites) return nullptr; *IsWrite = true; + *TypeSize = DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType()); *Alignment = SI->getAlignment(); - return SI->getPointerOperand(); - } - if (AtomicRMWInst *RMW = dyn_cast(I)) { + PtrOperand = SI->getPointerOperand(); + } else if (AtomicRMWInst *RMW = dyn_cast(I)) { if (!ClInstrumentAtomics) return nullptr; *IsWrite = true; + *TypeSize = DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType()); *Alignment = 0; - return RMW->getPointerOperand(); - } - if (AtomicCmpXchgInst *XCHG = dyn_cast(I)) { + PtrOperand = RMW->getPointerOperand(); + } else if (AtomicCmpXchgInst *XCHG = dyn_cast(I)) { if (!ClInstrumentAtomics) return nullptr; *IsWrite = true; + *TypeSize = DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType()); *Alignment = 0; - return XCHG->getPointerOperand(); + PtrOperand = XCHG->getPointerOperand(); } - return nullptr; + + // Treat memory accesses to promotable allocas as non-interesting since they + // will not cause memory violations. This greatly speeds up the instrumented + // executable at -O0. + if (ClSkipPromotableAllocas) + if (auto AI = dyn_cast_or_null(PtrOperand)) + return isInterestingAlloca(*AI) ? AI : nullptr; + + return PtrOperand; } static bool isPointerOperand(Value *V) { @@ -641,29 +905,25 @@ static bool isPointerOperand(Value *V) { // the frontend. static bool isInterestingPointerComparisonOrSubtraction(Instruction *I) { if (ICmpInst *Cmp = dyn_cast(I)) { - if (!Cmp->isRelational()) - return false; + if (!Cmp->isRelational()) return false; } else if (BinaryOperator *BO = dyn_cast(I)) { - if (BO->getOpcode() != Instruction::Sub) - return false; + if (BO->getOpcode() != Instruction::Sub) return false; } else { return false; } - if (!isPointerOperand(I->getOperand(0)) || - !isPointerOperand(I->getOperand(1))) - return false; - return true; + return isPointerOperand(I->getOperand(0)) && + isPointerOperand(I->getOperand(1)); } bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) { // If a global variable does not have dynamic initialization we don't // have to instrument it. However, if a global does not have initializer // at all, we assume it has dynamic initializer (in other TU). - return G->hasInitializer() && !DynamicallyInitializedGlobals.Contains(G); + return G->hasInitializer() && !GlobalsMD.get(G).IsDynInit; } -void -AddressSanitizer::instrumentPointerComparisonOrSubtraction(Instruction *I) { +void AddressSanitizer::instrumentPointerComparisonOrSubtraction( + Instruction *I) { IRBuilder<> IRB(I); Function *F = isa(I) ? AsanPtrCmpFunction : AsanPtrSubFunction; Value *Param[2] = {I->getOperand(0), I->getOperand(1)}; @@ -671,41 +931,50 @@ AddressSanitizer::instrumentPointerComparisonOrSubtraction(Instruction *I) { if (Param[i]->getType()->isPointerTy()) Param[i] = IRB.CreatePointerCast(Param[i], IntptrTy); } - IRB.CreateCall2(F, Param[0], Param[1]); + IRB.CreateCall(F, Param); } -void AddressSanitizer::instrumentMop(Instruction *I, bool UseCalls) { +void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis, + Instruction *I, bool UseCalls, + const DataLayout &DL) { bool IsWrite = false; unsigned Alignment = 0; - Value *Addr = isInterestingMemoryAccess(I, &IsWrite, &Alignment); + uint64_t TypeSize = 0; + Value *Addr = isInterestingMemoryAccess(I, &IsWrite, &TypeSize, &Alignment); assert(Addr); + + // Optimization experiments. + // The experiments can be used to evaluate potential optimizations that remove + // instrumentation (assess false negatives). Instead of completely removing + // some instrumentation, you set Exp to a non-zero value (mask of optimization + // experiments that want to remove instrumentation of this instruction). + // If Exp is non-zero, this pass will emit special calls into runtime + // (e.g. __asan_report_exp_load1 instead of __asan_report_load1). These calls + // make runtime terminate the program in a special way (with a different + // exit status). Then you run the new compiler on a buggy corpus, collect + // the special terminations (ideally, you don't see them at all -- no false + // negatives) and make the decision on the optimization. + uint32_t Exp = ClForceExperiment; + if (ClOpt && ClOptGlobals) { - if (GlobalVariable *G = dyn_cast(Addr)) { - // If initialization order checking is disabled, a simple access to a - // dynamically initialized global is always valid. - if (!ClInitializers || GlobalIsLinkerInitialized(G)) { - NumOptimizedAccessesToGlobalVar++; - return; - } - } - ConstantExpr *CE = dyn_cast(Addr); - if (CE && CE->isGEPWithNoNotionalOverIndexing()) { - if (GlobalVariable *G = dyn_cast(CE->getOperand(0))) { - if (CE->getOperand(1)->isNullValue() && GlobalIsLinkerInitialized(G)) { - NumOptimizedAccessesToGlobalArray++; - return; - } - } + // If initialization order checking is disabled, a simple access to a + // dynamically initialized global is always valid. + GlobalVariable *G = dyn_cast(GetUnderlyingObject(Addr, DL)); + if (G && (!ClInitializers || GlobalIsLinkerInitialized(G)) && + isSafeAccess(ObjSizeVis, Addr, TypeSize)) { + NumOptimizedAccessesToGlobalVar++; + return; } } - Type *OrigPtrTy = Addr->getType(); - Type *OrigTy = cast(OrigPtrTy)->getElementType(); - - assert(OrigTy->isSized()); - uint32_t TypeSize = DL->getTypeStoreSizeInBits(OrigTy); - - assert((TypeSize % 8) == 0); + if (ClOpt && ClOptStack) { + // A direct inbounds access to a stack variable is always valid. + if (isa(GetUnderlyingObject(Addr, DL)) && + isSafeAccess(ObjSizeVis, Addr, TypeSize)) { + NumOptimizedAccessesToStackVar++; + return; + } + } if (IsWrite) NumInstrumentedWrites++; @@ -718,65 +987,57 @@ void AddressSanitizer::instrumentMop(Instruction *I, bool UseCalls) { if ((TypeSize == 8 || TypeSize == 16 || TypeSize == 32 || TypeSize == 64 || TypeSize == 128) && (Alignment >= Granularity || Alignment == 0 || Alignment >= TypeSize / 8)) - return instrumentAddress(I, I, Addr, TypeSize, IsWrite, nullptr, UseCalls); - // Instrument unusual size or unusual alignment. - // We can not do it with a single check, so we do 1-byte check for the first - // and the last bytes. We call __asan_report_*_n(addr, real_size) to be able - // to report the actual access size. - IRBuilder<> IRB(I); - Value *Size = ConstantInt::get(IntptrTy, TypeSize / 8); - Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); - if (UseCalls) { - IRB.CreateCall2(AsanMemoryAccessCallbackSized[IsWrite], AddrLong, Size); - } else { - Value *LastByte = IRB.CreateIntToPtr( - IRB.CreateAdd(AddrLong, ConstantInt::get(IntptrTy, TypeSize / 8 - 1)), - OrigPtrTy); - instrumentAddress(I, I, Addr, 8, IsWrite, Size, false); - instrumentAddress(I, I, LastByte, 8, IsWrite, Size, false); - } -} - -// Validate the result of Module::getOrInsertFunction called for an interface -// function of AddressSanitizer. If the instrumented module defines a function -// with the same name, their prototypes must match, otherwise -// getOrInsertFunction returns a bitcast. -static Function *checkInterfaceFunction(Constant *FuncOrBitcast) { - if (isa(FuncOrBitcast)) return cast(FuncOrBitcast); - FuncOrBitcast->dump(); - report_fatal_error("trying to redefine an AddressSanitizer " - "interface function"); + return instrumentAddress(I, I, Addr, TypeSize, IsWrite, nullptr, UseCalls, + Exp); + instrumentUnusualSizeOrAlignment(I, Addr, TypeSize, IsWrite, nullptr, + UseCalls, Exp); } -Instruction *AddressSanitizer::generateCrashCode( - Instruction *InsertBefore, Value *Addr, - bool IsWrite, size_t AccessSizeIndex, Value *SizeArgument) { +Instruction *AddressSanitizer::generateCrashCode(Instruction *InsertBefore, + Value *Addr, bool IsWrite, + size_t AccessSizeIndex, + Value *SizeArgument, + uint32_t Exp) { IRBuilder<> IRB(InsertBefore); - CallInst *Call = SizeArgument - ? IRB.CreateCall2(AsanErrorCallbackSized[IsWrite], Addr, SizeArgument) - : IRB.CreateCall(AsanErrorCallback[IsWrite][AccessSizeIndex], Addr); + Value *ExpVal = Exp == 0 ? nullptr : ConstantInt::get(IRB.getInt32Ty(), Exp); + CallInst *Call = nullptr; + if (SizeArgument) { + if (Exp == 0) + Call = IRB.CreateCall(AsanErrorCallbackSized[IsWrite][0], + {Addr, SizeArgument}); + else + Call = IRB.CreateCall(AsanErrorCallbackSized[IsWrite][1], + {Addr, SizeArgument, ExpVal}); + } else { + if (Exp == 0) + Call = + IRB.CreateCall(AsanErrorCallback[IsWrite][0][AccessSizeIndex], Addr); + else + Call = IRB.CreateCall(AsanErrorCallback[IsWrite][1][AccessSizeIndex], + {Addr, ExpVal}); + } // We don't do Call->setDoesNotReturn() because the BB already has // UnreachableInst at the end. // This EmptyAsm is required to avoid callback merge. - IRB.CreateCall(EmptyAsm); + IRB.CreateCall(EmptyAsm, {}); return Call; } Value *AddressSanitizer::createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong, - Value *ShadowValue, - uint32_t TypeSize) { + Value *ShadowValue, + uint32_t TypeSize) { size_t Granularity = 1 << Mapping.Scale; // Addr & (Granularity - 1) - Value *LastAccessedByte = IRB.CreateAnd( - AddrLong, ConstantInt::get(IntptrTy, Granularity - 1)); + Value *LastAccessedByte = + IRB.CreateAnd(AddrLong, ConstantInt::get(IntptrTy, Granularity - 1)); // (Addr & (Granularity - 1)) + size - 1 if (TypeSize / 8 > 1) LastAccessedByte = IRB.CreateAdd( LastAccessedByte, ConstantInt::get(IntptrTy, TypeSize / 8 - 1)); // (uint8_t) ((Addr & (Granularity-1)) + size - 1) - LastAccessedByte = IRB.CreateIntCast( - LastAccessedByte, ShadowValue->getType(), false); + LastAccessedByte = + IRB.CreateIntCast(LastAccessedByte, ShadowValue->getType(), false); // ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue return IRB.CreateICmpSGE(LastAccessedByte, ShadowValue); } @@ -784,54 +1045,92 @@ Value *AddressSanitizer::createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong, void AddressSanitizer::instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore, Value *Addr, uint32_t TypeSize, bool IsWrite, - Value *SizeArgument, bool UseCalls) { + Value *SizeArgument, bool UseCalls, + uint32_t Exp) { IRBuilder<> IRB(InsertBefore); Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize); if (UseCalls) { - IRB.CreateCall(AsanMemoryAccessCallback[IsWrite][AccessSizeIndex], - AddrLong); + if (Exp == 0) + IRB.CreateCall(AsanMemoryAccessCallback[IsWrite][0][AccessSizeIndex], + AddrLong); + else + IRB.CreateCall(AsanMemoryAccessCallback[IsWrite][1][AccessSizeIndex], + {AddrLong, ConstantInt::get(IRB.getInt32Ty(), Exp)}); return; } - Type *ShadowTy = IntegerType::get( - *C, std::max(8U, TypeSize >> Mapping.Scale)); + Type *ShadowTy = + IntegerType::get(*C, std::max(8U, TypeSize >> Mapping.Scale)); Type *ShadowPtrTy = PointerType::get(ShadowTy, 0); Value *ShadowPtr = memToShadow(AddrLong, IRB); Value *CmpVal = Constant::getNullValue(ShadowTy); - Value *ShadowValue = IRB.CreateLoad( - IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy)); + Value *ShadowValue = + IRB.CreateLoad(IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy)); Value *Cmp = IRB.CreateICmpNE(ShadowValue, CmpVal); size_t Granularity = 1 << Mapping.Scale; TerminatorInst *CrashTerm = nullptr; if (ClAlwaysSlowPath || (TypeSize < 8 * Granularity)) { - TerminatorInst *CheckTerm = - SplitBlockAndInsertIfThen(Cmp, InsertBefore, false); - assert(dyn_cast(CheckTerm)->isUnconditional()); + // We use branch weights for the slow path check, to indicate that the slow + // path is rarely taken. This seems to be the case for SPEC benchmarks. + TerminatorInst *CheckTerm = SplitBlockAndInsertIfThen( + Cmp, InsertBefore, false, MDBuilder(*C).createBranchWeights(1, 100000)); + assert(cast(CheckTerm)->isUnconditional()); BasicBlock *NextBB = CheckTerm->getSuccessor(0); IRB.SetInsertPoint(CheckTerm); Value *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeSize); - BasicBlock *CrashBlock = + if (Recover) { + CrashTerm = SplitBlockAndInsertIfThen(Cmp2, CheckTerm, false); + } else { + BasicBlock *CrashBlock = BasicBlock::Create(*C, "", NextBB->getParent(), NextBB); - CrashTerm = new UnreachableInst(*C, CrashBlock); - BranchInst *NewTerm = BranchInst::Create(CrashBlock, NextBB, Cmp2); - ReplaceInstWithInst(CheckTerm, NewTerm); + CrashTerm = new UnreachableInst(*C, CrashBlock); + BranchInst *NewTerm = BranchInst::Create(CrashBlock, NextBB, Cmp2); + ReplaceInstWithInst(CheckTerm, NewTerm); + } } else { - CrashTerm = SplitBlockAndInsertIfThen(Cmp, InsertBefore, true); + CrashTerm = SplitBlockAndInsertIfThen(Cmp, InsertBefore, !Recover); } - Instruction *Crash = generateCrashCode( - CrashTerm, AddrLong, IsWrite, AccessSizeIndex, SizeArgument); + Instruction *Crash = generateCrashCode(CrashTerm, AddrLong, IsWrite, + AccessSizeIndex, SizeArgument, Exp); Crash->setDebugLoc(OrigIns->getDebugLoc()); } +// Instrument unusual size or unusual alignment. +// We can not do it with a single check, so we do 1-byte check for the first +// and the last bytes. We call __asan_report_*_n(addr, real_size) to be able +// to report the actual access size. +void AddressSanitizer::instrumentUnusualSizeOrAlignment( + Instruction *I, Value *Addr, uint32_t TypeSize, bool IsWrite, + Value *SizeArgument, bool UseCalls, uint32_t Exp) { + IRBuilder<> IRB(I); + Value *Size = ConstantInt::get(IntptrTy, TypeSize / 8); + Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); + if (UseCalls) { + if (Exp == 0) + IRB.CreateCall(AsanMemoryAccessCallbackSized[IsWrite][0], + {AddrLong, Size}); + else + IRB.CreateCall(AsanMemoryAccessCallbackSized[IsWrite][1], + {AddrLong, Size, ConstantInt::get(IRB.getInt32Ty(), Exp)}); + } else { + Value *LastByte = IRB.CreateIntToPtr( + IRB.CreateAdd(AddrLong, ConstantInt::get(IntptrTy, TypeSize / 8 - 1)), + Addr->getType()); + instrumentAddress(I, I, Addr, 8, IsWrite, Size, false, Exp); + instrumentAddress(I, I, LastByte, 8, IsWrite, Size, false, Exp); + } +} + void AddressSanitizerModule::poisonOneInitializer(Function &GlobalInit, GlobalValue *ModuleName) { // Set up the arguments to our poison/unpoison functions. - IRBuilder<> IRB(GlobalInit.begin()->getFirstInsertionPt()); + IRBuilder<> IRB(&GlobalInit.front(), + GlobalInit.front().getFirstInsertionPt()); // Add a call to poison all external globals before the given function starts. Value *ModuleNameAddr = ConstantExpr::getPointerCast(ModuleName, IntptrTy); @@ -849,15 +1148,16 @@ void AddressSanitizerModule::createInitializerPoisonCalls( ConstantArray *CA = cast(GV->getInitializer()); for (Use &OP : CA->operands()) { - if (isa(OP)) - continue; + if (isa(OP)) continue; ConstantStruct *CS = cast(OP); // Must have a function or null ptr. - // (CS->getOperand(0) is the init priority.) - if (Function* F = dyn_cast(CS->getOperand(1))) { - if (F->getName() != kAsanModuleCtorName) - poisonOneInitializer(*F, ModuleName); + if (Function *F = dyn_cast(CS->getOperand(1))) { + if (F->getName() == kAsanModuleCtorName) continue; + ConstantInt *Priority = dyn_cast(CS->getOperand(0)); + // Don't instrument CTORs that will run before asan.module_ctor. + if (Priority->getLimitedValue() <= kAsanCtorAndDtorPriority) continue; + poisonOneInitializer(*F, ModuleName); } } } @@ -866,60 +1166,38 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) { Type *Ty = cast(G->getType())->getElementType(); DEBUG(dbgs() << "GLOBAL: " << *G << "\n"); - if (BL->isIn(*G)) return false; + if (GlobalsMD.get(G).IsBlacklisted) return false; if (!Ty->isSized()) return false; if (!G->hasInitializer()) return false; if (GlobalWasGeneratedByAsan(G)) return false; // Our own global. // Touch only those globals that will not be defined in other modules. - // Don't handle ODR type linkages since other modules may be built w/o asan. + // Don't handle ODR linkage types and COMDATs since other modules may be built + // without ASan. if (G->getLinkage() != GlobalVariable::ExternalLinkage && G->getLinkage() != GlobalVariable::PrivateLinkage && G->getLinkage() != GlobalVariable::InternalLinkage) return false; + if (G->hasComdat()) return false; // Two problems with thread-locals: // - The address of the main thread's copy can't be computed at link-time. // - Need to poison all copies, not just the main thread's one. - if (G->isThreadLocal()) - return false; + if (G->isThreadLocal()) return false; // For now, just ignore this Global if the alignment is large. if (G->getAlignment() > MinRedzoneSizeForGlobal()) return false; - // Ignore all the globals with the names starting with "\01L_OBJC_". - // Many of those are put into the .cstring section. The linker compresses - // that section by removing the spare \0s after the string terminator, so - // our redzones get broken. - if ((G->getName().find("\01L_OBJC_") == 0) || - (G->getName().find("\01l_OBJC_") == 0)) { - DEBUG(dbgs() << "Ignoring \\01L_OBJC_* global: " << *G << "\n"); - return false; - } - if (G->hasSection()) { StringRef Section(G->getSection()); - // Ignore the globals from the __OBJC section. The ObjC runtime assumes - // those conform to /usr/lib/objc/runtime.h, so we can't add redzones to - // them. - if (Section.startswith("__OBJC,") || - Section.startswith("__DATA, __objc_")) { - DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n"); - return false; - } - // See http://code.google.com/p/address-sanitizer/issues/detail?id=32 - // Constant CFString instances are compiled in the following way: - // -- the string buffer is emitted into - // __TEXT,__cstring,cstring_literals - // -- the constant NSConstantString structure referencing that buffer - // is placed into __DATA,__cfstring - // Therefore there's no point in placing redzones into __DATA,__cfstring. - // Moreover, it causes the linker to crash on OS X 10.7 - if (Section.startswith("__DATA,__cfstring")) { - DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n"); - return false; - } - // The linker merges the contents of cstring_literals and removes the - // trailing zeroes. - if (Section.startswith("__TEXT,__cstring,cstring_literals")) { - DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n"); + + // Globals from llvm.metadata aren't emitted, do not instrument them. + if (Section == "llvm.metadata") return false; + // Do not instrument globals from special LLVM sections. + if (Section.find("__llvm") != StringRef::npos) return false; + + // Do not instrument function pointers to initialization and termination + // routines: dynamic linker will not properly handle redzones. + if (Section.startswith(".preinit_array") || + Section.startswith(".init_array") || + Section.startswith(".fini_array")) { return false; } @@ -932,8 +1210,41 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) { return false; } - // Globals from llvm.metadata aren't emitted, do not instrument them. - if (Section == "llvm.metadata") return false; + if (TargetTriple.isOSBinFormatMachO()) { + StringRef ParsedSegment, ParsedSection; + unsigned TAA = 0, StubSize = 0; + bool TAAParsed; + std::string ErrorCode = MCSectionMachO::ParseSectionSpecifier( + Section, ParsedSegment, ParsedSection, TAA, TAAParsed, StubSize); + assert(ErrorCode.empty() && "Invalid section specifier."); + + // Ignore the globals from the __OBJC section. The ObjC runtime assumes + // those conform to /usr/lib/objc/runtime.h, so we can't add redzones to + // them. + if (ParsedSegment == "__OBJC" || + (ParsedSegment == "__DATA" && ParsedSection.startswith("__objc_"))) { + DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n"); + return false; + } + // See http://code.google.com/p/address-sanitizer/issues/detail?id=32 + // Constant CFString instances are compiled in the following way: + // -- the string buffer is emitted into + // __TEXT,__cstring,cstring_literals + // -- the constant NSConstantString structure referencing that buffer + // is placed into __DATA,__cfstring + // Therefore there's no point in placing redzones into __DATA,__cfstring. + // Moreover, it causes the linker to crash on OS X 10.7 + if (ParsedSegment == "__DATA" && ParsedSection == "__cfstring") { + DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n"); + return false; + } + // The linker merges the contents of cstring_literals and removes the + // trailing zeroes. + if (ParsedSegment == "__TEXT" && (TAA & MachO::S_CSTRING_LITERALS)) { + DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n"); + return false; + } + } } return true; @@ -942,38 +1253,32 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) { void AddressSanitizerModule::initializeCallbacks(Module &M) { IRBuilder<> IRB(*C); // Declare our poisoning and unpoisoning functions. - AsanPoisonGlobals = checkInterfaceFunction(M.getOrInsertFunction( - kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy, NULL)); + AsanPoisonGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy, nullptr)); AsanPoisonGlobals->setLinkage(Function::ExternalLinkage); - AsanUnpoisonGlobals = checkInterfaceFunction(M.getOrInsertFunction( - kAsanUnpoisonGlobalsName, IRB.getVoidTy(), NULL)); + AsanUnpoisonGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanUnpoisonGlobalsName, IRB.getVoidTy(), nullptr)); AsanUnpoisonGlobals->setLinkage(Function::ExternalLinkage); // Declare functions that register/unregister globals. - AsanRegisterGlobals = checkInterfaceFunction(M.getOrInsertFunction( - kAsanRegisterGlobalsName, IRB.getVoidTy(), - IntptrTy, IntptrTy, NULL)); + AsanRegisterGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanRegisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr)); AsanRegisterGlobals->setLinkage(Function::ExternalLinkage); - AsanUnregisterGlobals = checkInterfaceFunction(M.getOrInsertFunction( - kAsanUnregisterGlobalsName, - IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); + AsanUnregisterGlobals = checkSanitizerInterfaceFunction( + M.getOrInsertFunction(kAsanUnregisterGlobalsName, IRB.getVoidTy(), + IntptrTy, IntptrTy, nullptr)); AsanUnregisterGlobals->setLinkage(Function::ExternalLinkage); - AsanCovModuleInit = checkInterfaceFunction(M.getOrInsertFunction( - kAsanCovModuleInitName, - IRB.getVoidTy(), IntptrTy, NULL)); - AsanCovModuleInit->setLinkage(Function::ExternalLinkage); } // This function replaces all global variables with new variables that have // trailing redzones. It also creates a function that poisons // redzones and inserts this function into llvm.global_ctors. bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) { - DynamicallyInitializedGlobals.Init(M); + GlobalsMD.init(M); SmallVector GlobalsToChange; for (auto &G : M.globals()) { - if (ShouldInstrumentGlobal(&G)) - GlobalsToChange.push_back(&G); + if (ShouldInstrumentGlobal(&G)) GlobalsToChange.push_back(&G); } size_t n = GlobalsToChange.size(); @@ -986,10 +1291,11 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) { // const char *name; // const char *module_name; // size_t has_dynamic_init; + // void *source_location; // We initialize an array of such structures and pass it to a run-time call. - StructType *GlobalStructTy = StructType::get(IntptrTy, IntptrTy, - IntptrTy, IntptrTy, - IntptrTy, IntptrTy, NULL); + StructType *GlobalStructTy = + StructType::get(IntptrTy, IntptrTy, IntptrTy, IntptrTy, IntptrTy, + IntptrTy, IntptrTy, nullptr); SmallVector Initializers(n); bool HasDynamicallyInitializedGlobals = false; @@ -997,45 +1303,46 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) { // We shouldn't merge same module names, as this string serves as unique // module ID in runtime. GlobalVariable *ModuleName = createPrivateGlobalForString( - M, M.getModuleIdentifier(), /*AllowMerging*/false); + M, M.getModuleIdentifier(), /*AllowMerging*/ false); + auto &DL = M.getDataLayout(); for (size_t i = 0; i < n; i++) { static const uint64_t kMaxGlobalRedzone = 1 << 18; GlobalVariable *G = GlobalsToChange[i]; + + auto MD = GlobalsMD.get(G); + // Create string holding the global name (use global name from metadata + // if it's available, otherwise just write the name of global variable). + GlobalVariable *Name = createPrivateGlobalForString( + M, MD.Name.empty() ? G->getName() : MD.Name, + /*AllowMerging*/ true); + PointerType *PtrTy = cast(G->getType()); Type *Ty = PtrTy->getElementType(); - uint64_t SizeInBytes = DL->getTypeAllocSize(Ty); + uint64_t SizeInBytes = DL.getTypeAllocSize(Ty); uint64_t MinRZ = MinRedzoneSizeForGlobal(); // MinRZ <= RZ <= kMaxGlobalRedzone // and trying to make RZ to be ~ 1/4 of SizeInBytes. - uint64_t RZ = std::max(MinRZ, - std::min(kMaxGlobalRedzone, - (SizeInBytes / MinRZ / 4) * MinRZ)); + uint64_t RZ = std::max( + MinRZ, std::min(kMaxGlobalRedzone, (SizeInBytes / MinRZ / 4) * MinRZ)); uint64_t RightRedzoneSize = RZ; // Round up to MinRZ - if (SizeInBytes % MinRZ) - RightRedzoneSize += MinRZ - (SizeInBytes % MinRZ); + if (SizeInBytes % MinRZ) RightRedzoneSize += MinRZ - (SizeInBytes % MinRZ); assert(((RightRedzoneSize + SizeInBytes) % MinRZ) == 0); Type *RightRedZoneTy = ArrayType::get(IRB.getInt8Ty(), RightRedzoneSize); - // Determine whether this global should be poisoned in initialization. - bool GlobalHasDynamicInitializer = - DynamicallyInitializedGlobals.Contains(G); - - StructType *NewTy = StructType::get(Ty, RightRedZoneTy, NULL); - Constant *NewInitializer = ConstantStruct::get( - NewTy, G->getInitializer(), - Constant::getNullValue(RightRedZoneTy), NULL); - GlobalVariable *Name = - createPrivateGlobalForString(M, G->getName(), /*AllowMerging*/true); + StructType *NewTy = StructType::get(Ty, RightRedZoneTy, nullptr); + Constant *NewInitializer = + ConstantStruct::get(NewTy, G->getInitializer(), + Constant::getNullValue(RightRedZoneTy), nullptr); // Create a new global variable with enough space for a redzone. GlobalValue::LinkageTypes Linkage = G->getLinkage(); if (G->isConstant() && Linkage == GlobalValue::PrivateLinkage) Linkage = GlobalValue::InternalLinkage; - GlobalVariable *NewGlobal = new GlobalVariable( - M, NewTy, G->isConstant(), Linkage, - NewInitializer, "", G, G->getThreadLocalMode()); + GlobalVariable *NewGlobal = + new GlobalVariable(M, NewTy, G->isConstant(), Linkage, NewInitializer, + "", G, G->getThreadLocalMode()); NewGlobal->copyAttributesFrom(G); NewGlobal->setAlignment(MinRZ); @@ -1044,23 +1351,27 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) { Indices2[1] = IRB.getInt32(0); G->replaceAllUsesWith( - ConstantExpr::getGetElementPtr(NewGlobal, Indices2, true)); + ConstantExpr::getGetElementPtr(NewTy, NewGlobal, Indices2, true)); NewGlobal->takeName(G); G->eraseFromParent(); + Constant *SourceLoc; + if (!MD.SourceLoc.empty()) { + auto SourceLocGlobal = createPrivateGlobalForSourceLoc(M, MD.SourceLoc); + SourceLoc = ConstantExpr::getPointerCast(SourceLocGlobal, IntptrTy); + } else { + SourceLoc = ConstantInt::get(IntptrTy, 0); + } + Initializers[i] = ConstantStruct::get( - GlobalStructTy, - ConstantExpr::getPointerCast(NewGlobal, IntptrTy), + GlobalStructTy, ConstantExpr::getPointerCast(NewGlobal, IntptrTy), ConstantInt::get(IntptrTy, SizeInBytes), ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize), ConstantExpr::getPointerCast(Name, IntptrTy), ConstantExpr::getPointerCast(ModuleName, IntptrTy), - ConstantInt::get(IntptrTy, GlobalHasDynamicInitializer), - NULL); + ConstantInt::get(IntptrTy, MD.IsDynInit), SourceLoc, nullptr); - // Populate the first and last globals declared in this TU. - if (ClInitializers && GlobalHasDynamicInitializer) - HasDynamicallyInitializedGlobals = true; + if (ClInitializers && MD.IsDynInit) HasDynamicallyInitializedGlobals = true; DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n"); } @@ -1073,20 +1384,20 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) { // Create calls for poisoning before initializers run and unpoisoning after. if (HasDynamicallyInitializedGlobals) createInitializerPoisonCalls(M, ModuleName); - IRB.CreateCall2(AsanRegisterGlobals, - IRB.CreatePointerCast(AllGlobals, IntptrTy), - ConstantInt::get(IntptrTy, n)); + IRB.CreateCall(AsanRegisterGlobals, + {IRB.CreatePointerCast(AllGlobals, IntptrTy), + ConstantInt::get(IntptrTy, n)}); // We also need to unregister globals at the end, e.g. when a shared library // gets closed. - Function *AsanDtorFunction = Function::Create( - FunctionType::get(Type::getVoidTy(*C), false), - GlobalValue::InternalLinkage, kAsanModuleDtorName, &M); + Function *AsanDtorFunction = + Function::Create(FunctionType::get(Type::getVoidTy(*C), false), + GlobalValue::InternalLinkage, kAsanModuleDtorName, &M); BasicBlock *AsanDtorBB = BasicBlock::Create(*C, "", AsanDtorFunction); IRBuilder<> IRB_Dtor(ReturnInst::Create(*C, AsanDtorBB)); - IRB_Dtor.CreateCall2(AsanUnregisterGlobals, - IRB.CreatePointerCast(AllGlobals, IntptrTy), - ConstantInt::get(IntptrTy, n)); + IRB_Dtor.CreateCall(AsanUnregisterGlobals, + {IRB.CreatePointerCast(AllGlobals, IntptrTy), + ConstantInt::get(IntptrTy, n)}); appendToGlobalDtors(M, AsanDtorFunction, kAsanCtorAndDtorPriority); DEBUG(dbgs() << M); @@ -1094,84 +1405,79 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) { } bool AddressSanitizerModule::runOnModule(Module &M) { - DataLayoutPass *DLP = getAnalysisIfAvailable(); - if (!DLP) - return false; - DL = &DLP->getDataLayout(); - BL.reset(SpecialCaseList::createOrDie(BlacklistFile)); C = &(M.getContext()); - int LongSize = DL->getPointerSizeInBits(); + int LongSize = M.getDataLayout().getPointerSizeInBits(); IntptrTy = Type::getIntNTy(*C, LongSize); - Mapping = getShadowMapping(M, LongSize); + TargetTriple = Triple(M.getTargetTriple()); + Mapping = getShadowMapping(TargetTriple, LongSize, CompileKernel); initializeCallbacks(M); bool Changed = false; - Function *CtorFunc = M.getFunction(kAsanModuleCtorName); - assert(CtorFunc); - IRBuilder<> IRB(CtorFunc->getEntryBlock().getTerminator()); - - if (ClCoverage > 0) { - Function *CovFunc = M.getFunction(kAsanCovName); - int nCov = CovFunc ? CovFunc->getNumUses() : 0; - IRB.CreateCall(AsanCovModuleInit, ConstantInt::get(IntptrTy, nCov)); - Changed = true; + // TODO(glider): temporarily disabled globals instrumentation for KASan. + if (ClGlobals && !CompileKernel) { + Function *CtorFunc = M.getFunction(kAsanModuleCtorName); + assert(CtorFunc); + IRBuilder<> IRB(CtorFunc->getEntryBlock().getTerminator()); + Changed |= InstrumentGlobals(IRB, M); } - if (ClGlobals && !BL->isIn(M)) Changed |= InstrumentGlobals(IRB, M); - return Changed; } void AddressSanitizer::initializeCallbacks(Module &M) { IRBuilder<> IRB(*C); // Create __asan_report* callbacks. - for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) { - for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; - AccessSizeIndex++) { - // IsWrite and TypeSize are encoded in the function name. - std::string Suffix = - (AccessIsWrite ? "store" : "load") + itostr(1 << AccessSizeIndex); - AsanErrorCallback[AccessIsWrite][AccessSizeIndex] = - checkInterfaceFunction( - M.getOrInsertFunction(kAsanReportErrorTemplate + Suffix, - IRB.getVoidTy(), IntptrTy, NULL)); - AsanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] = - checkInterfaceFunction( - M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + Suffix, - IRB.getVoidTy(), IntptrTy, NULL)); + // IsWrite, TypeSize and Exp are encoded in the function name. + for (int Exp = 0; Exp < 2; Exp++) { + for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) { + const std::string TypeStr = AccessIsWrite ? "store" : "load"; + const std::string ExpStr = Exp ? "exp_" : ""; + const std::string SuffixStr = CompileKernel ? "N" : "_n"; + const std::string EndingStr = Recover ? "_noabort" : ""; + Type *ExpType = Exp ? Type::getInt32Ty(*C) : nullptr; + AsanErrorCallbackSized[AccessIsWrite][Exp] = + checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanReportErrorTemplate + ExpStr + TypeStr + SuffixStr + EndingStr, + IRB.getVoidTy(), IntptrTy, IntptrTy, ExpType, nullptr)); + AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] = + checkSanitizerInterfaceFunction(M.getOrInsertFunction( + ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr, + IRB.getVoidTy(), IntptrTy, IntptrTy, ExpType, nullptr)); + for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; + AccessSizeIndex++) { + const std::string Suffix = TypeStr + itostr(1 << AccessSizeIndex); + AsanErrorCallback[AccessIsWrite][Exp][AccessSizeIndex] = + checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanReportErrorTemplate + ExpStr + Suffix + EndingStr, + IRB.getVoidTy(), IntptrTy, ExpType, nullptr)); + AsanMemoryAccessCallback[AccessIsWrite][Exp][AccessSizeIndex] = + checkSanitizerInterfaceFunction(M.getOrInsertFunction( + ClMemoryAccessCallbackPrefix + ExpStr + Suffix + EndingStr, + IRB.getVoidTy(), IntptrTy, ExpType, nullptr)); + } } } - AsanErrorCallbackSized[0] = checkInterfaceFunction(M.getOrInsertFunction( - kAsanReportLoadN, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); - AsanErrorCallbackSized[1] = checkInterfaceFunction(M.getOrInsertFunction( - kAsanReportStoreN, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); - - AsanMemoryAccessCallbackSized[0] = checkInterfaceFunction( - M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "loadN", - IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); - AsanMemoryAccessCallbackSized[1] = checkInterfaceFunction( - M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "storeN", - IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); - - AsanMemmove = checkInterfaceFunction(M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, NULL)); - AsanMemcpy = checkInterfaceFunction(M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + "memcpy", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, NULL)); - AsanMemset = checkInterfaceFunction(M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + "memset", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy, NULL)); - - AsanHandleNoReturnFunc = checkInterfaceFunction( - M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy(), NULL)); - AsanCovFunction = checkInterfaceFunction(M.getOrInsertFunction( - kAsanCovName, IRB.getVoidTy(), NULL)); - AsanPtrCmpFunction = checkInterfaceFunction(M.getOrInsertFunction( - kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); - AsanPtrSubFunction = checkInterfaceFunction(M.getOrInsertFunction( - kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); + + const std::string MemIntrinCallbackPrefix = + CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix; + AsanMemmove = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + MemIntrinCallbackPrefix + "memmove", IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, nullptr)); + AsanMemcpy = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + MemIntrinCallbackPrefix + "memcpy", IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, nullptr)); + AsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy, nullptr)); + + AsanHandleNoReturnFunc = checkSanitizerInterfaceFunction( + M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy(), nullptr)); + + AsanPtrCmpFunction = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr)); + AsanPtrSubFunction = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr)); // We insert an empty inline asm after __asan_report* to avoid callback merge. EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), StringRef(""), StringRef(""), @@ -1181,31 +1487,22 @@ void AddressSanitizer::initializeCallbacks(Module &M) { // virtual bool AddressSanitizer::doInitialization(Module &M) { // Initialize the private fields. No one has accessed them before. - DataLayoutPass *DLP = getAnalysisIfAvailable(); - if (!DLP) - report_fatal_error("data layout missing"); - DL = &DLP->getDataLayout(); - DynamicallyInitializedGlobals.Init(M); + GlobalsMD.init(M); C = &(M.getContext()); - LongSize = DL->getPointerSizeInBits(); + LongSize = M.getDataLayout().getPointerSizeInBits(); IntptrTy = Type::getIntNTy(*C, LongSize); - - AsanCtorFunction = Function::Create( - FunctionType::get(Type::getVoidTy(*C), false), - GlobalValue::InternalLinkage, kAsanModuleCtorName, &M); - BasicBlock *AsanCtorBB = BasicBlock::Create(*C, "", AsanCtorFunction); - // call __asan_init in the module ctor. - IRBuilder<> IRB(ReturnInst::Create(*C, AsanCtorBB)); - AsanInitFunction = checkInterfaceFunction( - M.getOrInsertFunction(kAsanInitName, IRB.getVoidTy(), NULL)); - AsanInitFunction->setLinkage(Function::ExternalLinkage); - IRB.CreateCall(AsanInitFunction); - - Mapping = getShadowMapping(M, LongSize); - - appendToGlobalCtors(M, AsanCtorFunction, kAsanCtorAndDtorPriority); + TargetTriple = Triple(M.getTargetTriple()); + + if (!CompileKernel) { + std::tie(AsanCtorFunction, AsanInitFunction) = + createSanitizerCtorAndInitFunctions( + M, kAsanModuleCtorName, kAsanInitName, + /*InitArgTypes=*/{}, /*InitArgs=*/{}, kAsanVersionCheckName); + appendToGlobalCtors(M, AsanCtorFunction, kAsanCtorAndDtorPriority); + } + Mapping = getShadowMapping(TargetTriple, LongSize, CompileKernel); return true; } @@ -1218,79 +1515,39 @@ bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) { // We cannot just ignore these methods, because they may call other // instrumented functions. if (F.getName().find(" load]") != std::string::npos) { - IRBuilder<> IRB(F.begin()->begin()); - IRB.CreateCall(AsanInitFunction); + IRBuilder<> IRB(&F.front(), F.front().begin()); + IRB.CreateCall(AsanInitFunction, {}); return true; } return false; } -void AddressSanitizer::InjectCoverageAtBlock(Function &F, BasicBlock &BB) { - BasicBlock::iterator IP = BB.getFirstInsertionPt(), BE = BB.end(); - // Skip static allocas at the top of the entry block so they don't become - // dynamic when we split the block. If we used our optimized stack layout, - // then there will only be one alloca and it will come first. - for (; IP != BE; ++IP) { - AllocaInst *AI = dyn_cast(IP); - if (!AI || !AI->isStaticAlloca()) +void AddressSanitizer::markEscapedLocalAllocas(Function &F) { + // Find the one possible call to llvm.localescape and pre-mark allocas passed + // to it as uninteresting. This assumes we haven't started processing allocas + // yet. This check is done up front because iterating the use list in + // isInterestingAlloca would be algorithmically slower. + assert(ProcessedAllocas.empty() && "must process localescape before allocas"); + + // Try to get the declaration of llvm.localescape. If it's not in the module, + // we can exit early. + if (!F.getParent()->getFunction("llvm.localescape")) return; + + // Look for a call to llvm.localescape call in the entry block. It can't be in + // any other block. + for (Instruction &I : F.getEntryBlock()) { + IntrinsicInst *II = dyn_cast(&I); + if (II && II->getIntrinsicID() == Intrinsic::localescape) { + // We found a call. Mark all the allocas passed in as uninteresting. + for (Value *Arg : II->arg_operands()) { + AllocaInst *AI = dyn_cast(Arg->stripPointerCasts()); + assert(AI && AI->isStaticAlloca() && + "non-static alloca arg to localescape"); + ProcessedAllocas[AI] = false; + } break; + } } - - DebugLoc EntryLoc = IP->getDebugLoc().getFnDebugLoc(*C); - IRBuilder<> IRB(IP); - IRB.SetCurrentDebugLocation(EntryLoc); - Type *Int8Ty = IRB.getInt8Ty(); - GlobalVariable *Guard = new GlobalVariable( - *F.getParent(), Int8Ty, false, GlobalValue::PrivateLinkage, - Constant::getNullValue(Int8Ty), "__asan_gen_cov_" + F.getName()); - LoadInst *Load = IRB.CreateLoad(Guard); - Load->setAtomic(Monotonic); - Load->setAlignment(1); - Value *Cmp = IRB.CreateICmpEQ(Constant::getNullValue(Int8Ty), Load); - Instruction *Ins = SplitBlockAndInsertIfThen( - Cmp, IP, false, MDBuilder(*C).createBranchWeights(1, 100000)); - IRB.SetInsertPoint(Ins); - IRB.SetCurrentDebugLocation(EntryLoc); - // We pass &F to __sanitizer_cov. We could avoid this and rely on - // GET_CALLER_PC, but having the PC of the first instruction is just nice. - IRB.CreateCall(AsanCovFunction); - StoreInst *Store = IRB.CreateStore(ConstantInt::get(Int8Ty, 1), Guard); - Store->setAtomic(Monotonic); - Store->setAlignment(1); -} - -// Poor man's coverage that works with ASan. -// We create a Guard boolean variable with the same linkage -// as the function and inject this code into the entry block (-asan-coverage=1) -// or all blocks (-asan-coverage=2): -// if (*Guard) { -// __sanitizer_cov(&F); -// *Guard = 1; -// } -// The accesses to Guard are atomic. The rest of the logic is -// in __sanitizer_cov (it's fine to call it more than once). -// -// This coverage implementation provides very limited data: -// it only tells if a given function (block) was ever executed. -// No counters, no per-edge data. -// But for many use cases this is what we need and the added slowdown -// is negligible. This simple implementation will probably be obsoleted -// by the upcoming Clang-based coverage implementation. -// By having it here and now we hope to -// a) get the functionality to users earlier and -// b) collect usage statistics to help improve Clang coverage design. -bool AddressSanitizer::InjectCoverage(Function &F, - const ArrayRef AllBlocks) { - if (!ClCoverage) return false; - - if (ClCoverage == 1 || - (unsigned)ClCoverageBlockThreshold < AllBlocks.size()) { - InjectCoverageAtBlock(F, F.getEntryBlock()); - } else { - for (auto BB : AllBlocks) - InjectCoverageAtBlock(F, *BB); - } - return true; } bool AddressSanitizer::runOnFunction(Function &F) { @@ -1299,25 +1556,32 @@ bool AddressSanitizer::runOnFunction(Function &F) { DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n"); initializeCallbacks(*F.getParent()); + DT = &getAnalysis().getDomTree(); + // If needed, insert __asan_init before checking for SanitizeAddress attr. maybeInsertAsanInitAtFunctionEntry(F); - if (!F.hasFnAttribute(Attribute::SanitizeAddress)) - return false; + if (!F.hasFnAttribute(Attribute::SanitizeAddress)) return false; - if (!ClDebugFunc.empty() && ClDebugFunc != F.getName()) - return false; + if (!ClDebugFunc.empty() && ClDebugFunc != F.getName()) return false; + + FunctionStateRAII CleanupObj(this); + + // We can't instrument allocas used with llvm.localescape. Only static allocas + // can be passed to that intrinsic. + markEscapedLocalAllocas(F); // We want to instrument every address only once per basic block (unless there // are calls between uses). - SmallSet TempsToInstrument; - SmallVector ToInstrument; - SmallVector NoReturnCalls; - SmallVector AllBlocks; - SmallVector PointerComparisonsOrSubtracts; + SmallSet TempsToInstrument; + SmallVector ToInstrument; + SmallVector NoReturnCalls; + SmallVector AllBlocks; + SmallVector PointerComparisonsOrSubtracts; int NumAllocas = 0; bool IsWrite; unsigned Alignment; + uint64_t TypeSize; // Fill the set of memory operations to instrument. for (auto &BB : F) { @@ -1326,10 +1590,10 @@ bool AddressSanitizer::runOnFunction(Function &F) { int NumInsnsPerBB = 0; for (auto &Inst : BB) { if (LooksLikeCodeInBug11395(&Inst)) return false; - if (Value *Addr = - isInterestingMemoryAccess(&Inst, &IsWrite, &Alignment)) { + if (Value *Addr = isInterestingMemoryAccess(&Inst, &IsWrite, &TypeSize, + &Alignment)) { if (ClOpt && ClOptSameTemp) { - if (!TempsToInstrument.insert(Addr)) + if (!TempsToInstrument.insert(Addr).second) continue; // We've seen this temp in the current BB. } } else if (ClInvalidPointerPairs && @@ -1339,47 +1603,39 @@ bool AddressSanitizer::runOnFunction(Function &F) { } else if (isa(Inst)) { // ok, take it. } else { - if (isa(Inst)) - NumAllocas++; + if (isa(Inst)) NumAllocas++; CallSite CS(&Inst); if (CS) { // A call inside BB. TempsToInstrument.clear(); - if (CS.doesNotReturn()) - NoReturnCalls.push_back(CS.getInstruction()); + if (CS.doesNotReturn()) NoReturnCalls.push_back(CS.getInstruction()); } continue; } ToInstrument.push_back(&Inst); NumInsnsPerBB++; - if (NumInsnsPerBB >= ClMaxInsnsToInstrumentPerBB) - break; + if (NumInsnsPerBB >= ClMaxInsnsToInstrumentPerBB) break; } } - Function *UninstrumentedDuplicate = nullptr; - bool LikelyToInstrument = - !NoReturnCalls.empty() || !ToInstrument.empty() || (NumAllocas > 0); - if (ClKeepUninstrumented && LikelyToInstrument) { - ValueToValueMapTy VMap; - UninstrumentedDuplicate = CloneFunction(&F, VMap, false); - UninstrumentedDuplicate->removeFnAttr(Attribute::SanitizeAddress); - UninstrumentedDuplicate->setName("NOASAN_" + F.getName()); - F.getParent()->getFunctionList().push_back(UninstrumentedDuplicate); - } - - bool UseCalls = false; - if (ClInstrumentationWithCallsThreshold >= 0 && - ToInstrument.size() > (unsigned)ClInstrumentationWithCallsThreshold) - UseCalls = true; + bool UseCalls = + CompileKernel || + (ClInstrumentationWithCallsThreshold >= 0 && + ToInstrument.size() > (unsigned)ClInstrumentationWithCallsThreshold); + const TargetLibraryInfo *TLI = + &getAnalysis().getTLI(); + const DataLayout &DL = F.getParent()->getDataLayout(); + ObjectSizeOffsetVisitor ObjSizeVis(DL, TLI, F.getContext(), + /*RoundToAlign=*/true); // Instrument. int NumInstrumented = 0; for (auto Inst : ToInstrument) { if (ClDebugMin < 0 || ClDebugMax < 0 || (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) { - if (isInterestingMemoryAccess(Inst, &IsWrite, &Alignment)) - instrumentMop(Inst, UseCalls); + if (isInterestingMemoryAccess(Inst, &IsWrite, &TypeSize, &Alignment)) + instrumentMop(ObjSizeVis, Inst, UseCalls, + F.getParent()->getDataLayout()); else instrumentMemIntrinsic(cast(Inst)); } @@ -1393,7 +1649,7 @@ bool AddressSanitizer::runOnFunction(Function &F) { // See e.g. http://code.google.com/p/address-sanitizer/issues/detail?id=37 for (auto CI : NoReturnCalls) { IRBuilder<> IRB(CI); - IRB.CreateCall(AsanHandleNoReturnFunc); + IRB.CreateCall(AsanHandleNoReturnFunc, {}); } for (auto Inst : PointerComparisonsOrSubtracts) { @@ -1403,25 +1659,8 @@ bool AddressSanitizer::runOnFunction(Function &F) { bool res = NumInstrumented > 0 || ChangedStack || !NoReturnCalls.empty(); - if (InjectCoverage(F, AllBlocks)) - res = true; - DEBUG(dbgs() << "ASAN done instrumenting: " << res << " " << F << "\n"); - if (ClKeepUninstrumented) { - if (!res) { - // No instrumentation is done, no need for the duplicate. - if (UninstrumentedDuplicate) - UninstrumentedDuplicate->eraseFromParent(); - } else { - // The function was instrumented. We must have the duplicate. - assert(UninstrumentedDuplicate); - UninstrumentedDuplicate->setSection("NOASAN"); - assert(!F.hasSection()); - F.setSection("ASAN"); - } - } - return res; } @@ -1441,23 +1680,29 @@ void FunctionStackPoisoner::initializeCallbacks(Module &M) { IRBuilder<> IRB(*C); for (int i = 0; i <= kMaxAsanStackMallocSizeClass; i++) { std::string Suffix = itostr(i); - AsanStackMallocFunc[i] = checkInterfaceFunction( + AsanStackMallocFunc[i] = checkSanitizerInterfaceFunction( M.getOrInsertFunction(kAsanStackMallocNameTemplate + Suffix, IntptrTy, - IntptrTy, IntptrTy, NULL)); - AsanStackFreeFunc[i] = checkInterfaceFunction(M.getOrInsertFunction( - kAsanStackFreeNameTemplate + Suffix, IRB.getVoidTy(), IntptrTy, - IntptrTy, IntptrTy, NULL)); + IntptrTy, nullptr)); + AsanStackFreeFunc[i] = checkSanitizerInterfaceFunction( + M.getOrInsertFunction(kAsanStackFreeNameTemplate + Suffix, + IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr)); } - AsanPoisonStackMemoryFunc = checkInterfaceFunction(M.getOrInsertFunction( - kAsanPoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); - AsanUnpoisonStackMemoryFunc = checkInterfaceFunction(M.getOrInsertFunction( - kAsanUnpoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL)); + AsanPoisonStackMemoryFunc = checkSanitizerInterfaceFunction( + M.getOrInsertFunction(kAsanPoisonStackMemoryName, IRB.getVoidTy(), + IntptrTy, IntptrTy, nullptr)); + AsanUnpoisonStackMemoryFunc = checkSanitizerInterfaceFunction( + M.getOrInsertFunction(kAsanUnpoisonStackMemoryName, IRB.getVoidTy(), + IntptrTy, IntptrTy, nullptr)); + AsanAllocaPoisonFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanAllocaPoison, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr)); + AsanAllocasUnpoisonFunc = + checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kAsanAllocasUnpoison, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr)); } -void -FunctionStackPoisoner::poisonRedZones(const ArrayRef ShadowBytes, - IRBuilder<> &IRB, Value *ShadowBase, - bool DoPoison) { +void FunctionStackPoisoner::poisonRedZones(ArrayRef ShadowBytes, + IRBuilder<> &IRB, Value *ShadowBase, + bool DoPoison) { size_t n = ShadowBytes.size(); size_t i = 0; // We need to (un)poison n bytes of stack shadow. Poison as many as we can @@ -1468,7 +1713,7 @@ FunctionStackPoisoner::poisonRedZones(const ArrayRef ShadowBytes, for (; i + LargeStoreSizeInBytes - 1 < n; i += LargeStoreSizeInBytes) { uint64_t Val = 0; for (size_t j = 0; j < LargeStoreSizeInBytes; j++) { - if (ASan.DL->isLittleEndian()) + if (F.getParent()->getDataLayout().isLittleEndian()) Val |= (uint64_t)ShadowBytes[i + j] << (8 * j); else Val = (Val << 8) | ShadowBytes[i + j]; @@ -1487,9 +1732,8 @@ FunctionStackPoisoner::poisonRedZones(const ArrayRef ShadowBytes, static int StackMallocSizeClass(uint64_t LocalStackSize) { assert(LocalStackSize <= kMaxStackMallocSize); uint64_t MaxSize = kMinStackMallocSize; - for (int i = 0; ; i++, MaxSize *= 2) - if (LocalStackSize <= MaxSize) - return i; + for (int i = 0;; i++, MaxSize *= 2) + if (LocalStackSize <= MaxSize) return i; llvm_unreachable("impossible LocalStackSize"); } @@ -1501,36 +1745,107 @@ static int StackMallocSizeClass(uint64_t LocalStackSize) { void FunctionStackPoisoner::SetShadowToStackAfterReturnInlined( IRBuilder<> &IRB, Value *ShadowBase, int Size) { assert(!(Size % 8)); - assert(kAsanStackAfterReturnMagic == 0xf5); + + // kAsanStackAfterReturnMagic is 0xf5. + const uint64_t kAsanStackAfterReturnMagic64 = 0xf5f5f5f5f5f5f5f5ULL; + for (int i = 0; i < Size; i += 8) { Value *p = IRB.CreateAdd(ShadowBase, ConstantInt::get(IntptrTy, i)); - IRB.CreateStore(ConstantInt::get(IRB.getInt64Ty(), 0xf5f5f5f5f5f5f5f5ULL), - IRB.CreateIntToPtr(p, IRB.getInt64Ty()->getPointerTo())); + IRB.CreateStore( + ConstantInt::get(IRB.getInt64Ty(), kAsanStackAfterReturnMagic64), + IRB.CreateIntToPtr(p, IRB.getInt64Ty()->getPointerTo())); + } +} + +PHINode *FunctionStackPoisoner::createPHI(IRBuilder<> &IRB, Value *Cond, + Value *ValueIfTrue, + Instruction *ThenTerm, + Value *ValueIfFalse) { + PHINode *PHI = IRB.CreatePHI(IntptrTy, 2); + BasicBlock *CondBlock = cast(Cond)->getParent(); + PHI->addIncoming(ValueIfFalse, CondBlock); + BasicBlock *ThenBlock = ThenTerm->getParent(); + PHI->addIncoming(ValueIfTrue, ThenBlock); + return PHI; +} + +Value *FunctionStackPoisoner::createAllocaForLayout( + IRBuilder<> &IRB, const ASanStackFrameLayout &L, bool Dynamic) { + AllocaInst *Alloca; + if (Dynamic) { + Alloca = IRB.CreateAlloca(IRB.getInt8Ty(), + ConstantInt::get(IRB.getInt64Ty(), L.FrameSize), + "MyAlloca"); + } else { + Alloca = IRB.CreateAlloca(ArrayType::get(IRB.getInt8Ty(), L.FrameSize), + nullptr, "MyAlloca"); + assert(Alloca->isStaticAlloca()); } + assert((ClRealignStack & (ClRealignStack - 1)) == 0); + size_t FrameAlignment = std::max(L.FrameAlignment, (size_t)ClRealignStack); + Alloca->setAlignment(FrameAlignment); + return IRB.CreatePointerCast(Alloca, IntptrTy); } -static DebugLoc getFunctionEntryDebugLocation(Function &F) { - for (const auto &Inst : F.getEntryBlock()) - if (!isa(Inst)) - return Inst.getDebugLoc(); - return DebugLoc(); +void FunctionStackPoisoner::createDynamicAllocasInitStorage() { + BasicBlock &FirstBB = *F.begin(); + IRBuilder<> IRB(dyn_cast(FirstBB.begin())); + DynamicAllocaLayout = IRB.CreateAlloca(IntptrTy, nullptr); + IRB.CreateStore(Constant::getNullValue(IntptrTy), DynamicAllocaLayout); + DynamicAllocaLayout->setAlignment(32); } void FunctionStackPoisoner::poisonStack() { + assert(AllocaVec.size() > 0 || DynamicAllocaVec.size() > 0); + + // Insert poison calls for lifetime intrinsics for alloca. + bool HavePoisonedAllocas = false; + for (const auto &APC : AllocaPoisonCallVec) { + assert(APC.InsBefore); + assert(APC.AI); + IRBuilder<> IRB(APC.InsBefore); + poisonAlloca(APC.AI, APC.Size, IRB, APC.DoPoison); + HavePoisonedAllocas |= APC.DoPoison; + } + + if (ClInstrumentAllocas && DynamicAllocaVec.size() > 0) { + // Handle dynamic allocas. + createDynamicAllocasInitStorage(); + for (auto &AI : DynamicAllocaVec) handleDynamicAllocaCall(AI); + + unpoisonDynamicAllocas(); + } + + if (AllocaVec.empty()) return; + int StackMallocIdx = -1; - DebugLoc EntryDebugLocation = getFunctionEntryDebugLocation(F); + DebugLoc EntryDebugLocation; + if (auto SP = getDISubprogram(&F)) + EntryDebugLocation = DebugLoc::get(SP->getScopeLine(), 0, SP); - assert(AllocaVec.size() > 0); Instruction *InsBefore = AllocaVec[0]; IRBuilder<> IRB(InsBefore); IRB.SetCurrentDebugLocation(EntryDebugLocation); + // Make sure non-instrumented allocas stay in the entry block. Otherwise, + // debug info is broken, because only entry-block allocas are treated as + // regular stack slots. + auto InsBeforeB = InsBefore->getParent(); + assert(InsBeforeB == &F.getEntryBlock()); + for (BasicBlock::iterator I(InsBefore); I != InsBeforeB->end(); ++I) + if (auto *AI = dyn_cast(I)) + if (NonInstrumentedStaticAllocaVec.count(AI) > 0) + AI->moveBefore(InsBefore); + + // If we have a call to llvm.localescape, keep it in the entry block. + if (LocalEscapeCall) LocalEscapeCall->moveBefore(InsBefore); + SmallVector SVD; SVD.reserve(AllocaVec.size()); for (AllocaInst *AI : AllocaVec) { - ASanStackVariableDescription D = { AI->getName().data(), - getAllocaSizeInBytes(AI), - AI->getAlignment(), AI, 0}; + ASanStackVariableDescription D = {AI->getName().data(), + ASan.getAllocaSizeInBytes(AI), + AI->getAlignment(), AI, 0}; SVD.push_back(D); } // Minimal header size (left redzone) is 4 pointers, @@ -1540,54 +1855,64 @@ void FunctionStackPoisoner::poisonStack() { ComputeASanStackFrameLayout(SVD, 1UL << Mapping.Scale, MinHeaderSize, &L); DEBUG(dbgs() << L.DescriptionString << " --- " << L.FrameSize << "\n"); uint64_t LocalStackSize = L.FrameSize; - bool DoStackMalloc = - ClUseAfterReturn && LocalStackSize <= kMaxStackMallocSize; - - Type *ByteArrayTy = ArrayType::get(IRB.getInt8Ty(), LocalStackSize); - AllocaInst *MyAlloca = - new AllocaInst(ByteArrayTy, "MyAlloca", InsBefore); - MyAlloca->setDebugLoc(EntryDebugLocation); - assert((ClRealignStack & (ClRealignStack - 1)) == 0); - size_t FrameAlignment = std::max(L.FrameAlignment, (size_t)ClRealignStack); - MyAlloca->setAlignment(FrameAlignment); - assert(MyAlloca->isStaticAlloca()); - Value *OrigStackBase = IRB.CreatePointerCast(MyAlloca, IntptrTy); - Value *LocalStackBase = OrigStackBase; + bool DoStackMalloc = ClUseAfterReturn && !ASan.CompileKernel && + LocalStackSize <= kMaxStackMallocSize; + bool DoDynamicAlloca = ClDynamicAllocaStack; + // Don't do dynamic alloca or stack malloc if: + // 1) There is inline asm: too often it makes assumptions on which registers + // are available. + // 2) There is a returns_twice call (typically setjmp), which is + // optimization-hostile, and doesn't play well with introduced indirect + // register-relative calculation of local variable addresses. + DoDynamicAlloca &= !HasNonEmptyInlineAsm && !HasReturnsTwiceCall; + DoStackMalloc &= !HasNonEmptyInlineAsm && !HasReturnsTwiceCall; + + Value *StaticAlloca = + DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, false); + + Value *FakeStack; + Value *LocalStackBase; if (DoStackMalloc) { - // LocalStackBase = OrigStackBase - // if (__asan_option_detect_stack_use_after_return) - // LocalStackBase = __asan_stack_malloc_N(LocalStackBase, OrigStackBase); - StackMallocIdx = StackMallocSizeClass(LocalStackSize); - assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass); + // void *FakeStack = __asan_option_detect_stack_use_after_return + // ? __asan_stack_malloc_N(LocalStackSize) + // : nullptr; + // void *LocalStackBase = (FakeStack) ? FakeStack : alloca(LocalStackSize); Constant *OptionDetectUAR = F.getParent()->getOrInsertGlobal( kAsanOptionDetectUAR, IRB.getInt32Ty()); - Value *Cmp = IRB.CreateICmpNE(IRB.CreateLoad(OptionDetectUAR), - Constant::getNullValue(IRB.getInt32Ty())); - Instruction *Term = SplitBlockAndInsertIfThen(Cmp, InsBefore, false); - BasicBlock *CmpBlock = cast(Cmp)->getParent(); + Value *UARIsEnabled = + IRB.CreateICmpNE(IRB.CreateLoad(OptionDetectUAR), + Constant::getNullValue(IRB.getInt32Ty())); + Instruction *Term = + SplitBlockAndInsertIfThen(UARIsEnabled, InsBefore, false); IRBuilder<> IRBIf(Term); IRBIf.SetCurrentDebugLocation(EntryDebugLocation); - LocalStackBase = IRBIf.CreateCall2( - AsanStackMallocFunc[StackMallocIdx], - ConstantInt::get(IntptrTy, LocalStackSize), OrigStackBase); - BasicBlock *SetBlock = cast(LocalStackBase)->getParent(); + StackMallocIdx = StackMallocSizeClass(LocalStackSize); + assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass); + Value *FakeStackValue = + IRBIf.CreateCall(AsanStackMallocFunc[StackMallocIdx], + ConstantInt::get(IntptrTy, LocalStackSize)); IRB.SetInsertPoint(InsBefore); IRB.SetCurrentDebugLocation(EntryDebugLocation); - PHINode *Phi = IRB.CreatePHI(IntptrTy, 2); - Phi->addIncoming(OrigStackBase, CmpBlock); - Phi->addIncoming(LocalStackBase, SetBlock); - LocalStackBase = Phi; - } + FakeStack = createPHI(IRB, UARIsEnabled, FakeStackValue, Term, + ConstantInt::get(IntptrTy, 0)); - // Insert poison calls for lifetime intrinsics for alloca. - bool HavePoisonedAllocas = false; - for (const auto &APC : AllocaPoisonCallVec) { - assert(APC.InsBefore); - assert(APC.AI); - IRBuilder<> IRB(APC.InsBefore); - poisonAlloca(APC.AI, APC.Size, IRB, APC.DoPoison); - HavePoisonedAllocas |= APC.DoPoison; + Value *NoFakeStack = + IRB.CreateICmpEQ(FakeStack, Constant::getNullValue(IntptrTy)); + Term = SplitBlockAndInsertIfThen(NoFakeStack, InsBefore, false); + IRBIf.SetInsertPoint(Term); + IRBIf.SetCurrentDebugLocation(EntryDebugLocation); + Value *AllocaValue = + DoDynamicAlloca ? createAllocaForLayout(IRBIf, L, true) : StaticAlloca; + IRB.SetInsertPoint(InsBefore); + IRB.SetCurrentDebugLocation(EntryDebugLocation); + LocalStackBase = createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStack); + } else { + // void *FakeStack = nullptr; + // void *LocalStackBase = alloca(LocalStackSize); + FakeStack = ConstantInt::get(IntptrTy, 0); + LocalStackBase = + DoDynamicAlloca ? createAllocaForLayout(IRB, L, true) : StaticAlloca; } // Replace Alloca instructions with base+offset. @@ -1596,7 +1921,7 @@ void FunctionStackPoisoner::poisonStack() { Value *NewAllocaPtr = IRB.CreateIntToPtr( IRB.CreateAdd(LocalStackBase, ConstantInt::get(IntptrTy, Desc.Offset)), AI->getType()); - replaceDbgDeclareForAlloca(AI, NewAllocaPtr, DIB); + replaceDbgDeclareForAlloca(AI, NewAllocaPtr, DIB, /*Deref=*/true); AI->replaceAllUsesWith(NewAllocaPtr); } @@ -1607,19 +1932,19 @@ void FunctionStackPoisoner::poisonStack() { BasePlus0); // Write the frame description constant to redzone[1]. Value *BasePlus1 = IRB.CreateIntToPtr( - IRB.CreateAdd(LocalStackBase, ConstantInt::get(IntptrTy, ASan.LongSize/8)), - IntptrPtrTy); + IRB.CreateAdd(LocalStackBase, + ConstantInt::get(IntptrTy, ASan.LongSize / 8)), + IntptrPtrTy); GlobalVariable *StackDescriptionGlobal = createPrivateGlobalForString(*F.getParent(), L.DescriptionString, - /*AllowMerging*/true); - Value *Description = IRB.CreatePointerCast(StackDescriptionGlobal, - IntptrTy); + /*AllowMerging*/ true); + Value *Description = IRB.CreatePointerCast(StackDescriptionGlobal, IntptrTy); IRB.CreateStore(Description, BasePlus1); // Write the PC to redzone[2]. Value *BasePlus2 = IRB.CreateIntToPtr( - IRB.CreateAdd(LocalStackBase, ConstantInt::get(IntptrTy, - 2 * ASan.LongSize/8)), - IntptrPtrTy); + IRB.CreateAdd(LocalStackBase, + ConstantInt::get(IntptrTy, 2 * ASan.LongSize / 8)), + IntptrPtrTy); IRB.CreateStore(IRB.CreatePointerCast(&F, IntptrTy), BasePlus2); // Poison the stack redzones at the entry. @@ -1634,17 +1959,18 @@ void FunctionStackPoisoner::poisonStack() { BasePlus0); if (DoStackMalloc) { assert(StackMallocIdx >= 0); - // if LocalStackBase != OrigStackBase: + // if FakeStack != 0 // LocalStackBase == FakeStack // // In use-after-return mode, poison the whole stack frame. // if StackMallocIdx <= 4 // // For small sizes inline the whole thing: // memset(ShadowBase, kAsanStackAfterReturnMagic, ShadowSize); - // **SavedFlagPtr(LocalStackBase) = 0 + // **SavedFlagPtr(FakeStack) = 0 // else - // __asan_stack_free_N(LocalStackBase, OrigStackBase) + // __asan_stack_free_N(FakeStack, LocalStackSize) // else // - Value *Cmp = IRBRet.CreateICmpNE(LocalStackBase, OrigStackBase); + Value *Cmp = + IRBRet.CreateICmpNE(FakeStack, Constant::getNullValue(IntptrTy)); TerminatorInst *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, Ret, &ThenTerm, &ElseTerm); @@ -1654,7 +1980,7 @@ void FunctionStackPoisoner::poisonStack() { SetShadowToStackAfterReturnInlined(IRBPoison, ShadowBase, ClassSize >> Mapping.Scale); Value *SavedFlagPtrPtr = IRBPoison.CreateAdd( - LocalStackBase, + FakeStack, ConstantInt::get(IntptrTy, ClassSize - ASan.LongSize / 8)); Value *SavedFlagPtr = IRBPoison.CreateLoad( IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy)); @@ -1663,9 +1989,9 @@ void FunctionStackPoisoner::poisonStack() { IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getInt8PtrTy())); } else { // For larger frames call __asan_stack_free_*. - IRBPoison.CreateCall3(AsanStackFreeFunc[StackMallocIdx], LocalStackBase, - ConstantInt::get(IntptrTy, LocalStackSize), - OrigStackBase); + IRBPoison.CreateCall( + AsanStackFreeFunc[StackMallocIdx], + {FakeStack, ConstantInt::get(IntptrTy, LocalStackSize)}); } IRBuilder<> IRBElse(ElseTerm); @@ -1673,7 +1999,6 @@ void FunctionStackPoisoner::poisonStack() { } else if (HavePoisonedAllocas) { // If we poisoned some allocas in llvm.lifetime analysis, // unpoison whole stack frame now. - assert(LocalStackBase == OrigStackBase); poisonAlloca(LocalStackBase, LocalStackSize, IRBRet, false); } else { poisonRedZones(L.ShadowBytes, IRBRet, ShadowBase, false); @@ -1681,8 +2006,7 @@ void FunctionStackPoisoner::poisonStack() { } // We are done. Remove the old unused alloca instructions. - for (auto AI : AllocaVec) - AI->eraseFromParent(); + for (auto AI : AllocaVec) AI->eraseFromParent(); } void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size, @@ -1690,9 +2014,9 @@ void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size, // For now just insert the call to ASan runtime. Value *AddrArg = IRB.CreatePointerCast(V, IntptrTy); Value *SizeArg = ConstantInt::get(IntptrTy, Size); - IRB.CreateCall2(DoPoison ? AsanPoisonStackMemoryFunc - : AsanUnpoisonStackMemoryFunc, - AddrArg, SizeArg); + IRB.CreateCall( + DoPoison ? AsanPoisonStackMemoryFunc : AsanUnpoisonStackMemoryFunc, + {AddrArg, SizeArg}); } // Handling llvm.lifetime intrinsics for a given %alloca: @@ -1707,12 +2031,11 @@ void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size, AllocaInst *FunctionStackPoisoner::findAllocaForValue(Value *V) { if (AllocaInst *AI = dyn_cast(V)) // We're intested only in allocas we can handle. - return isInterestingAlloca(*AI) ? AI : nullptr; + return ASan.isInterestingAlloca(*AI) ? AI : nullptr; // See if we've already calculated (or started to calculate) alloca for a // given value. AllocaForValueMapTy::iterator I = AllocaForValue.find(V); - if (I != AllocaForValue.end()) - return I->second; + if (I != AllocaForValue.end()) return I->second; // Store 0 while we're calculating alloca for value V to avoid // infinite recursion if the value references itself. AllocaForValue[V] = nullptr; @@ -1720,8 +2043,7 @@ AllocaInst *FunctionStackPoisoner::findAllocaForValue(Value *V) { if (CastInst *CI = dyn_cast(V)) Res = findAllocaForValue(CI->getOperand(0)); else if (PHINode *PN = dyn_cast(V)) { - for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { - Value *IncValue = PN->getIncomingValue(i); + for (Value *IncValue : PN->incoming_values()) { // Allow self-referencing phi-nodes. if (IncValue == PN) continue; AllocaInst *IncValueAI = findAllocaForValue(IncValue); @@ -1731,7 +2053,85 @@ AllocaInst *FunctionStackPoisoner::findAllocaForValue(Value *V) { Res = IncValueAI; } } - if (Res) - AllocaForValue[V] = Res; + if (Res) AllocaForValue[V] = Res; return Res; } + +void FunctionStackPoisoner::handleDynamicAllocaCall(AllocaInst *AI) { + IRBuilder<> IRB(AI); + + const unsigned Align = std::max(kAllocaRzSize, AI->getAlignment()); + const uint64_t AllocaRedzoneMask = kAllocaRzSize - 1; + + Value *Zero = Constant::getNullValue(IntptrTy); + Value *AllocaRzSize = ConstantInt::get(IntptrTy, kAllocaRzSize); + Value *AllocaRzMask = ConstantInt::get(IntptrTy, AllocaRedzoneMask); + + // Since we need to extend alloca with additional memory to locate + // redzones, and OldSize is number of allocated blocks with + // ElementSize size, get allocated memory size in bytes by + // OldSize * ElementSize. + const unsigned ElementSize = + F.getParent()->getDataLayout().getTypeAllocSize(AI->getAllocatedType()); + Value *OldSize = + IRB.CreateMul(IRB.CreateIntCast(AI->getArraySize(), IntptrTy, false), + ConstantInt::get(IntptrTy, ElementSize)); + + // PartialSize = OldSize % 32 + Value *PartialSize = IRB.CreateAnd(OldSize, AllocaRzMask); + + // Misalign = kAllocaRzSize - PartialSize; + Value *Misalign = IRB.CreateSub(AllocaRzSize, PartialSize); + + // PartialPadding = Misalign != kAllocaRzSize ? Misalign : 0; + Value *Cond = IRB.CreateICmpNE(Misalign, AllocaRzSize); + Value *PartialPadding = IRB.CreateSelect(Cond, Misalign, Zero); + + // AdditionalChunkSize = Align + PartialPadding + kAllocaRzSize + // Align is added to locate left redzone, PartialPadding for possible + // partial redzone and kAllocaRzSize for right redzone respectively. + Value *AdditionalChunkSize = IRB.CreateAdd( + ConstantInt::get(IntptrTy, Align + kAllocaRzSize), PartialPadding); + + Value *NewSize = IRB.CreateAdd(OldSize, AdditionalChunkSize); + + // Insert new alloca with new NewSize and Align params. + AllocaInst *NewAlloca = IRB.CreateAlloca(IRB.getInt8Ty(), NewSize); + NewAlloca->setAlignment(Align); + + // NewAddress = Address + Align + Value *NewAddress = IRB.CreateAdd(IRB.CreatePtrToInt(NewAlloca, IntptrTy), + ConstantInt::get(IntptrTy, Align)); + + // Insert __asan_alloca_poison call for new created alloca. + IRB.CreateCall(AsanAllocaPoisonFunc, {NewAddress, OldSize}); + + // Store the last alloca's address to DynamicAllocaLayout. We'll need this + // for unpoisoning stuff. + IRB.CreateStore(IRB.CreatePtrToInt(NewAlloca, IntptrTy), DynamicAllocaLayout); + + Value *NewAddressPtr = IRB.CreateIntToPtr(NewAddress, AI->getType()); + + // Replace all uses of AddessReturnedByAlloca with NewAddressPtr. + AI->replaceAllUsesWith(NewAddressPtr); + + // We are done. Erase old alloca from parent. + AI->eraseFromParent(); +} + +// isSafeAccess returns true if Addr is always inbounds with respect to its +// base object. For example, it is a field access or an array access with +// constant inbounds index. +bool AddressSanitizer::isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis, + Value *Addr, uint64_t TypeSize) const { + SizeOffsetType SizeOffset = ObjSizeVis.compute(Addr); + if (!ObjSizeVis.bothKnown(SizeOffset)) return false; + uint64_t Size = SizeOffset.first.getZExtValue(); + int64_t Offset = SizeOffset.second.getSExtValue(); + // Three checks are required to ensure safety: + // . Offset >= 0 (since the offset is given from the base ptr) + // . Size >= Offset (unsigned) + // . Size - Offset >= NeededSize (unsigned) + return Offset >= 0 && Size >= uint64_t(Offset) && + Size - uint64_t(Offset) >= TypeSize / 8; +}