1 //===-- CDSPass.cpp - xxx -------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 // This file is distributed under the University of Illinois Open Source
7 // License. See LICENSE.TXT for details.
9 //===----------------------------------------------------------------------===//
11 // This file is a modified version of ThreadSanitizer.cpp, a part of a race detector.
13 // The tool is under development, for the details about previous versions see
14 // http://code.google.com/p/data-race-test
16 // The instrumentation phase is quite simple:
17 // - Insert calls to run-time library before every memory access.
18 // - Optimizations may apply to avoid instrumenting some of the accesses.
19 // - Insert calls at function entry/exit.
20 // The rest is handled by the run-time library.
21 //===----------------------------------------------------------------------===//
23 #include "llvm/ADT/Statistic.h"
24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/ADT/SmallString.h"
26 #include "llvm/Analysis/ValueTracking.h"
27 #include "llvm/Analysis/CaptureTracking.h"
28 #include "llvm/IR/BasicBlock.h"
29 #include "llvm/IR/Function.h"
30 #include "llvm/IR/IRBuilder.h"
31 #include "llvm/IR/Instructions.h"
32 #include "llvm/IR/IntrinsicInst.h"
33 #include "llvm/IR/LLVMContext.h"
34 #include "llvm/IR/LegacyPassManager.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/PassManager.h"
37 #include "llvm/Pass.h"
38 #include "llvm/ProfileData/InstrProf.h"
39 #include "llvm/Support/raw_ostream.h"
40 #include "llvm/Support/AtomicOrdering.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Transforms/Scalar.h"
43 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
44 #include "llvm/Transforms/Utils/EscapeEnumerator.h"
45 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
50 #define DEBUG_TYPE "CDS"
51 #include <llvm/IR/DebugLoc.h>
53 Value *getPosition( Instruction * I, IRBuilder <> IRB, bool print = false)
55 const DebugLoc & debug_location = I->getDebugLoc ();
56 std::string position_string;
58 llvm::raw_string_ostream position_stream (position_string);
59 debug_location . print (position_stream);
63 errs() << position_string << "\n";
66 return IRB.CreateGlobalStringPtr (position_string);
69 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
70 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
71 STATISTIC(NumOmittedReadsBeforeWrite,
72 "Number of reads ignored due to following writes");
73 STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size");
74 // STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes");
75 // STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads");
76 STATISTIC(NumOmittedReadsFromConstantGlobals,
77 "Number of reads from constant globals");
78 STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads");
79 STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing");
81 // static const char *const kCDSModuleCtorName = "cds.module_ctor";
82 // static const char *const kCDSInitName = "cds_init";
93 static const size_t kNumberOfAccessSizes = 4;
95 int getAtomicOrderIndex(AtomicOrdering order) {
97 case AtomicOrdering::Monotonic:
98 return (int)AtomicOrderingCABI::relaxed;
99 // case AtomicOrdering::Consume: // not specified yet
100 // return AtomicOrderingCABI::consume;
101 case AtomicOrdering::Acquire:
102 return (int)AtomicOrderingCABI::acquire;
103 case AtomicOrdering::Release:
104 return (int)AtomicOrderingCABI::release;
105 case AtomicOrdering::AcquireRelease:
106 return (int)AtomicOrderingCABI::acq_rel;
107 case AtomicOrdering::SequentiallyConsistent:
108 return (int)AtomicOrderingCABI::seq_cst;
110 // unordered or Not Atomic
115 AtomicOrderingCABI indexToAtomicOrder(int index) {
118 return AtomicOrderingCABI::relaxed;
120 return AtomicOrderingCABI::consume;
122 return AtomicOrderingCABI::acquire;
124 return AtomicOrderingCABI::release;
126 return AtomicOrderingCABI::acq_rel;
128 return AtomicOrderingCABI::seq_cst;
130 errs() << "Bad Atomic index\n";
131 return AtomicOrderingCABI::seq_cst;
135 /* According to atomic_base.h: __cmpexch_failure_order */
136 int AtomicCasFailureOrderIndex(int index) {
137 AtomicOrderingCABI succ_order = indexToAtomicOrder(index);
138 AtomicOrderingCABI fail_order;
139 if (succ_order == AtomicOrderingCABI::acq_rel)
140 fail_order = AtomicOrderingCABI::acquire;
141 else if (succ_order == AtomicOrderingCABI::release)
142 fail_order = AtomicOrderingCABI::relaxed;
144 fail_order = succ_order;
146 return (int) fail_order;
149 /* The original function checkSanitizerInterfaceFunction was defined
150 * in llvm/Transforms/Utils/ModuleUtils.h
152 static Function * checkCDSPassInterfaceFunction(Constant *FuncOrBitcast) {
153 if (isa<Function>(FuncOrBitcast))
154 return cast<Function>(FuncOrBitcast);
155 FuncOrBitcast->print(errs());
158 raw_string_ostream Stream(Err);
159 Stream << "CDSPass interface function redefined: " << *FuncOrBitcast;
160 report_fatal_error(Err);
164 struct CDSPass : public FunctionPass {
165 CDSPass() : FunctionPass(ID) {}
166 StringRef getPassName() const override;
167 bool runOnFunction(Function &F) override;
168 bool doInitialization(Module &M) override;
172 void initializeCallbacks(Module &M);
173 bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL);
174 bool instrumentVolatile(Instruction *I, const DataLayout &DL);
175 bool instrumentMemIntrinsic(Instruction *I);
176 bool isAtomicCall(Instruction *I);
177 bool instrumentAtomic(Instruction *I, const DataLayout &DL);
178 bool instrumentAtomicCall(CallInst *CI, const DataLayout &DL);
179 bool shouldInstrumentBeforeAtomics(Instruction *I);
180 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local,
181 SmallVectorImpl<Instruction *> &All,
182 const DataLayout &DL);
183 bool addrPointsToConstantData(Value *Addr);
184 int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL);
186 Function * CDSFuncEntry;
187 Function * CDSFuncExit;
189 Function * CDSLoad[kNumberOfAccessSizes];
190 Function * CDSStore[kNumberOfAccessSizes];
191 Function * CDSVolatileLoad[kNumberOfAccessSizes];
192 Function * CDSVolatileStore[kNumberOfAccessSizes];
193 Function * CDSAtomicInit[kNumberOfAccessSizes];
194 Function * CDSAtomicLoad[kNumberOfAccessSizes];
195 Function * CDSAtomicStore[kNumberOfAccessSizes];
196 Function * CDSAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes];
197 Function * CDSAtomicCAS_V1[kNumberOfAccessSizes];
198 Function * CDSAtomicCAS_V2[kNumberOfAccessSizes];
199 Function * CDSAtomicThreadFence;
200 Function * MemmoveFn, * MemcpyFn, * MemsetFn;
201 // Function * CDSCtorFunction;
203 std::vector<StringRef> AtomicFuncNames;
204 std::vector<StringRef> PartialAtomicFuncNames;
208 StringRef CDSPass::getPassName() const {
212 void CDSPass::initializeCallbacks(Module &M) {
213 LLVMContext &Ctx = M.getContext();
215 Attr = Attr.addAttribute(Ctx, AttributeList::FunctionIndex,
216 Attribute::NoUnwind);
218 Type * Int1Ty = Type::getInt1Ty(Ctx);
219 Type * Int32Ty = Type::getInt32Ty(Ctx);
220 OrdTy = Type::getInt32Ty(Ctx);
222 Int8PtrTy = Type::getInt8PtrTy(Ctx);
223 Int16PtrTy = Type::getInt16PtrTy(Ctx);
224 Int32PtrTy = Type::getInt32PtrTy(Ctx);
225 Int64PtrTy = Type::getInt64PtrTy(Ctx);
227 VoidTy = Type::getVoidTy(Ctx);
229 CDSFuncEntry = checkCDSPassInterfaceFunction(
230 M.getOrInsertFunction("cds_func_entry",
231 Attr, VoidTy, Int8PtrTy));
232 CDSFuncExit = checkCDSPassInterfaceFunction(
233 M.getOrInsertFunction("cds_func_exit",
234 Attr, VoidTy, Int8PtrTy));
236 // Get the function to call from our untime library.
237 for (unsigned i = 0; i < kNumberOfAccessSizes; i++) {
238 const unsigned ByteSize = 1U << i;
239 const unsigned BitSize = ByteSize * 8;
241 std::string ByteSizeStr = utostr(ByteSize);
242 std::string BitSizeStr = utostr(BitSize);
244 Type *Ty = Type::getIntNTy(Ctx, BitSize);
245 Type *PtrTy = Ty->getPointerTo();
247 // uint8_t cds_atomic_load8 (void * obj, int atomic_index)
248 // void cds_atomic_store8 (void * obj, int atomic_index, uint8_t val)
249 SmallString<32> LoadName("cds_load" + BitSizeStr);
250 SmallString<32> StoreName("cds_store" + BitSizeStr);
251 SmallString<32> VolatileLoadName("cds_volatile_load" + BitSizeStr);
252 SmallString<32> VolatileStoreName("cds_volatile_store" + BitSizeStr);
253 SmallString<32> AtomicInitName("cds_atomic_init" + BitSizeStr);
254 SmallString<32> AtomicLoadName("cds_atomic_load" + BitSizeStr);
255 SmallString<32> AtomicStoreName("cds_atomic_store" + BitSizeStr);
257 CDSLoad[i] = checkCDSPassInterfaceFunction(
258 M.getOrInsertFunction(LoadName, Attr, VoidTy, PtrTy));
259 CDSStore[i] = checkCDSPassInterfaceFunction(
260 M.getOrInsertFunction(StoreName, Attr, VoidTy, PtrTy));
261 CDSVolatileLoad[i] = checkCDSPassInterfaceFunction(
262 M.getOrInsertFunction(VolatileLoadName,
263 Attr, Ty, PtrTy, Int8PtrTy));
264 CDSVolatileStore[i] = checkCDSPassInterfaceFunction(
265 M.getOrInsertFunction(VolatileStoreName,
266 Attr, VoidTy, PtrTy, Ty, Int8PtrTy));
267 CDSAtomicInit[i] = checkCDSPassInterfaceFunction(
268 M.getOrInsertFunction(AtomicInitName,
269 Attr, VoidTy, PtrTy, Ty, Int8PtrTy));
270 CDSAtomicLoad[i] = checkCDSPassInterfaceFunction(
271 M.getOrInsertFunction(AtomicLoadName,
272 Attr, Ty, PtrTy, OrdTy, Int8PtrTy));
273 CDSAtomicStore[i] = checkCDSPassInterfaceFunction(
274 M.getOrInsertFunction(AtomicStoreName,
275 Attr, VoidTy, PtrTy, Ty, OrdTy, Int8PtrTy));
277 for (int op = AtomicRMWInst::FIRST_BINOP;
278 op <= AtomicRMWInst::LAST_BINOP; ++op) {
279 CDSAtomicRMW[op][i] = nullptr;
280 std::string NamePart;
282 if (op == AtomicRMWInst::Xchg)
283 NamePart = "_exchange";
284 else if (op == AtomicRMWInst::Add)
285 NamePart = "_fetch_add";
286 else if (op == AtomicRMWInst::Sub)
287 NamePart = "_fetch_sub";
288 else if (op == AtomicRMWInst::And)
289 NamePart = "_fetch_and";
290 else if (op == AtomicRMWInst::Or)
291 NamePart = "_fetch_or";
292 else if (op == AtomicRMWInst::Xor)
293 NamePart = "_fetch_xor";
297 SmallString<32> AtomicRMWName("cds_atomic" + NamePart + BitSizeStr);
298 CDSAtomicRMW[op][i] = checkCDSPassInterfaceFunction(
299 M.getOrInsertFunction(AtomicRMWName,
300 Attr, Ty, PtrTy, Ty, OrdTy, Int8PtrTy));
303 // only supportes strong version
304 SmallString<32> AtomicCASName_V1("cds_atomic_compare_exchange" + BitSizeStr + "_v1");
305 SmallString<32> AtomicCASName_V2("cds_atomic_compare_exchange" + BitSizeStr + "_v2");
306 CDSAtomicCAS_V1[i] = checkCDSPassInterfaceFunction(
307 M.getOrInsertFunction(AtomicCASName_V1,
308 Attr, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, Int8PtrTy));
309 CDSAtomicCAS_V2[i] = checkCDSPassInterfaceFunction(
310 M.getOrInsertFunction(AtomicCASName_V2,
311 Attr, Int1Ty, PtrTy, PtrTy, Ty, OrdTy, OrdTy, Int8PtrTy));
314 CDSAtomicThreadFence = checkCDSPassInterfaceFunction(
315 M.getOrInsertFunction("cds_atomic_thread_fence", Attr, VoidTy, OrdTy, Int8PtrTy));
317 MemmoveFn = checkCDSPassInterfaceFunction(
318 M.getOrInsertFunction("memmove", Attr, Int8PtrTy, Int8PtrTy,
319 Int8PtrTy, IntPtrTy));
320 MemcpyFn = checkCDSPassInterfaceFunction(
321 M.getOrInsertFunction("memcpy", Attr, Int8PtrTy, Int8PtrTy,
322 Int8PtrTy, IntPtrTy));
323 MemsetFn = checkCDSPassInterfaceFunction(
324 M.getOrInsertFunction("memset", Attr, Int8PtrTy, Int8PtrTy,
328 bool CDSPass::doInitialization(Module &M) {
329 const DataLayout &DL = M.getDataLayout();
330 IntPtrTy = DL.getIntPtrType(M.getContext());
332 // createSanitizerCtorAndInitFunctions is defined in "llvm/Transforms/Utils/ModuleUtils.h"
333 // We do not support it yet
335 std::tie(CDSCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions(
336 M, kCDSModuleCtorName, kCDSInitName, {}, {});
338 appendToGlobalCtors(M, CDSCtorFunction, 0);
343 "atomic_init", "atomic_load", "atomic_store",
344 "atomic_fetch_", "atomic_exchange", "atomic_compare_exchange_"
347 PartialAtomicFuncNames =
349 "load", "store", "fetch", "exchange", "compare_exchange_"
355 static bool isVtableAccess(Instruction *I) {
356 if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa))
357 return Tag->isTBAAVtableAccess();
361 // Do not instrument known races/"benign races" that come from compiler
362 // instrumentatin. The user has no way of suppressing them.
363 static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) {
364 // Peel off GEPs and BitCasts.
365 Addr = Addr->stripInBoundsOffsets();
367 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
368 if (GV->hasSection()) {
369 StringRef SectionName = GV->getSection();
370 // Check if the global is in the PGO counters section.
371 auto OF = Triple(M->getTargetTriple()).getObjectFormat();
372 if (SectionName.endswith(
373 getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
377 // Check if the global is private gcov data.
378 if (GV->getName().startswith("__llvm_gcov") ||
379 GV->getName().startswith("__llvm_gcda"))
383 // Do not instrument acesses from different address spaces; we cannot deal
386 Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType());
387 if (PtrTy->getPointerAddressSpace() != 0)
394 bool CDSPass::addrPointsToConstantData(Value *Addr) {
395 // If this is a GEP, just analyze its pointer operand.
396 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr))
397 Addr = GEP->getPointerOperand();
399 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
400 if (GV->isConstant()) {
401 // Reads from constant globals can not race with any writes.
402 NumOmittedReadsFromConstantGlobals++;
405 } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) {
406 if (isVtableAccess(L)) {
407 // Reads from a vtable pointer can not race with any writes.
408 NumOmittedReadsFromVtable++;
415 bool CDSPass::shouldInstrumentBeforeAtomics(Instruction * Inst) {
416 if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
417 AtomicOrdering ordering = LI->getOrdering();
418 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
420 } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
421 AtomicOrdering ordering = SI->getOrdering();
422 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
424 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst)) {
425 AtomicOrdering ordering = RMWI->getOrdering();
426 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
428 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(Inst)) {
429 AtomicOrdering ordering = CASI->getSuccessOrdering();
430 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
432 } else if (FenceInst *FI = dyn_cast<FenceInst>(Inst)) {
433 AtomicOrdering ordering = FI->getOrdering();
434 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
441 void CDSPass::chooseInstructionsToInstrument(
442 SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All,
443 const DataLayout &DL) {
444 SmallPtrSet<Value*, 8> WriteTargets;
445 // Iterate from the end.
446 for (Instruction *I : reverse(Local)) {
447 if (StoreInst *Store = dyn_cast<StoreInst>(I)) {
448 Value *Addr = Store->getPointerOperand();
449 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr))
451 WriteTargets.insert(Addr);
453 LoadInst *Load = cast<LoadInst>(I);
454 Value *Addr = Load->getPointerOperand();
455 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr))
457 if (WriteTargets.count(Addr)) {
458 // We will write to this temp, so no reason to analyze the read.
459 NumOmittedReadsBeforeWrite++;
462 if (addrPointsToConstantData(Addr)) {
463 // Addr points to some constant data -- it can not race with any writes.
467 Value *Addr = isa<StoreInst>(*I)
468 ? cast<StoreInst>(I)->getPointerOperand()
469 : cast<LoadInst>(I)->getPointerOperand();
470 if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) &&
471 !PointerMayBeCaptured(Addr, true, true)) {
472 // The variable is addressable but not captured, so it cannot be
473 // referenced from a different thread and participate in a data race
474 // (see llvm/Analysis/CaptureTracking.h for details).
475 NumOmittedNonCaptured++;
484 void CDSPass::InsertRuntimeIgnores(Function &F) {
485 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
486 IRB.CreateCall(CDSIgnoreBegin);
487 EscapeEnumerator EE(F, "cds_ignore_cleanup", ClHandleCxxExceptions);
488 while (IRBuilder<> *AtExit = EE.Next()) {
489 AtExit->CreateCall(CDSIgnoreEnd);
493 bool CDSPass::runOnFunction(Function &F) {
494 if (F.getName() == "main") {
495 F.setName("user_main");
496 errs() << "main replaced by user_main\n";
499 initializeCallbacks( *F.getParent() );
500 SmallVector<Instruction*, 8> AllLoadsAndStores;
501 SmallVector<Instruction*, 8> LocalLoadsAndStores;
502 SmallVector<Instruction*, 8> VolatileLoadsAndStores;
503 SmallVector<Instruction*, 8> AtomicAccesses;
504 SmallVector<Instruction*, 8> MemIntrinCalls;
507 bool HasAtomic = false;
508 bool HasVolatile = false;
509 const DataLayout &DL = F.getParent()->getDataLayout();
512 for (auto &Inst : BB) {
513 if ( (&Inst)->isAtomic() ) {
514 AtomicAccesses.push_back(&Inst);
517 if (shouldInstrumentBeforeAtomics(&Inst)) {
518 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores,
521 } else if (isAtomicCall(&Inst) ) {
522 AtomicAccesses.push_back(&Inst);
524 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores,
526 } else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) {
527 LoadInst *LI = dyn_cast<LoadInst>(&Inst);
528 StoreInst *SI = dyn_cast<StoreInst>(&Inst);
529 bool isVolatile = ( LI ? LI->isVolatile() : SI->isVolatile() );
532 VolatileLoadsAndStores.push_back(&Inst);
535 LocalLoadsAndStores.push_back(&Inst);
536 } else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) {
537 if (isa<MemIntrinsic>(Inst))
538 MemIntrinCalls.push_back(&Inst);
540 /*if (CallInst *CI = dyn_cast<CallInst>(&Inst))
541 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI);
544 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores,
549 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL);
552 for (auto Inst : AllLoadsAndStores) {
553 Res |= instrumentLoadOrStore(Inst, DL);
556 for (auto Inst : VolatileLoadsAndStores) {
557 Res |= instrumentVolatile(Inst, DL);
560 for (auto Inst : AtomicAccesses) {
561 Res |= instrumentAtomic(Inst, DL);
564 for (auto Inst : MemIntrinCalls) {
565 Res |= instrumentMemIntrinsic(Inst);
568 // Only instrument functions that contain atomics or volatiles
569 if (Res && ( HasAtomic || HasVolatile) ) {
570 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
572 Value *ReturnAddress = IRB.CreateCall(
573 Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress),
577 Value * FuncName = IRB.CreateGlobalStringPtr(F.getName());
578 IRB.CreateCall(CDSFuncEntry, FuncName);
580 EscapeEnumerator EE(F, "cds_cleanup", true);
581 while (IRBuilder<> *AtExit = EE.Next()) {
582 AtExit->CreateCall(CDSFuncExit, FuncName);
591 bool CDSPass::instrumentLoadOrStore(Instruction *I,
592 const DataLayout &DL) {
594 bool IsWrite = isa<StoreInst>(*I);
595 Value *Addr = IsWrite
596 ? cast<StoreInst>(I)->getPointerOperand()
597 : cast<LoadInst>(I)->getPointerOperand();
599 // swifterror memory addresses are mem2reg promoted by instruction selection.
600 // As such they cannot have regular uses like an instrumentation function and
601 // it makes no sense to track them as memory.
602 if (Addr->isSwiftError())
605 int Idx = getMemoryAccessFuncIndex(Addr, DL);
609 if (IsWrite && isVtableAccess(I)) {
611 LLVM_DEBUG(dbgs() << " VPTR : " << *I << "\n");
612 Value *StoredValue = cast<StoreInst>(I)->getValueOperand();
613 // StoredValue may be a vector type if we are storing several vptrs at once.
614 // In this case, just take the first element of the vector since this is
615 // enough to find vptr races.
616 if (isa<VectorType>(StoredValue->getType()))
617 StoredValue = IRB.CreateExtractElement(
618 StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0));
619 if (StoredValue->getType()->isIntegerTy())
620 StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy());
621 // Call TsanVptrUpdate.
622 IRB.CreateCall(TsanVptrUpdate,
623 {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
624 IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())});
625 NumInstrumentedVtableWrites++;
630 if (!IsWrite && isVtableAccess(I)) {
632 IRB.CreateCall(TsanVptrLoad,
633 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()));
634 NumInstrumentedVtableReads++;
639 // TODO: unaligned reads and writes
641 Value *OnAccessFunc = nullptr;
642 OnAccessFunc = IsWrite ? CDSStore[Idx] : CDSLoad[Idx];
644 Type *ArgType = IRB.CreatePointerCast(Addr, Addr->getType())->getType();
646 if ( ArgType != Int8PtrTy && ArgType != Int16PtrTy &&
647 ArgType != Int32PtrTy && ArgType != Int64PtrTy ) {
648 // if other types of load or stores are passed in
652 IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, Addr->getType()));
653 if (IsWrite) NumInstrumentedWrites++;
654 else NumInstrumentedReads++;
658 bool CDSPass::instrumentVolatile(Instruction * I, const DataLayout &DL) {
660 Value *position = getPosition(I, IRB);
662 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
663 assert( LI->isVolatile() );
664 Value *Addr = LI->getPointerOperand();
665 int Idx=getMemoryAccessFuncIndex(Addr, DL);
669 Value *args[] = {Addr, position};
670 Instruction* funcInst = CallInst::Create(CDSVolatileLoad[Idx], args);
671 ReplaceInstWithInst(LI, funcInst);
672 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
673 assert( SI->isVolatile() );
674 Value *Addr = SI->getPointerOperand();
675 int Idx=getMemoryAccessFuncIndex(Addr, DL);
679 Value *val = SI->getValueOperand();
680 Value *args[] = {Addr, val, position};
681 Instruction* funcInst = CallInst::Create(CDSVolatileStore[Idx], args);
682 ReplaceInstWithInst(SI, funcInst);
690 bool CDSPass::instrumentMemIntrinsic(Instruction *I) {
692 if (MemSetInst *M = dyn_cast<MemSetInst>(I)) {
695 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()),
696 IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false),
697 IRB.CreateIntCast(M->getArgOperand(2), IntPtrTy, false)});
698 I->eraseFromParent();
699 } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) {
701 isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn,
702 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()),
703 IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()),
704 IRB.CreateIntCast(M->getArgOperand(2), IntPtrTy, false)});
705 I->eraseFromParent();
710 bool CDSPass::instrumentAtomic(Instruction * I, const DataLayout &DL) {
713 if (auto *CI = dyn_cast<CallInst>(I)) {
714 return instrumentAtomicCall(CI, DL);
717 Value *position = getPosition(I, IRB);
719 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
720 Value *Addr = LI->getPointerOperand();
721 int Idx=getMemoryAccessFuncIndex(Addr, DL);
725 int atomic_order_index = getAtomicOrderIndex(LI->getOrdering());
726 Value *order = ConstantInt::get(OrdTy, atomic_order_index);
727 Value *args[] = {Addr, order, position};
728 Instruction* funcInst = CallInst::Create(CDSAtomicLoad[Idx], args);
729 ReplaceInstWithInst(LI, funcInst);
730 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
731 Value *Addr = SI->getPointerOperand();
732 int Idx=getMemoryAccessFuncIndex(Addr, DL);
736 int atomic_order_index = getAtomicOrderIndex(SI->getOrdering());
737 Value *val = SI->getValueOperand();
738 Value *order = ConstantInt::get(OrdTy, atomic_order_index);
739 Value *args[] = {Addr, val, order, position};
740 Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
741 ReplaceInstWithInst(SI, funcInst);
742 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) {
743 Value *Addr = RMWI->getPointerOperand();
744 int Idx=getMemoryAccessFuncIndex(Addr, DL);
748 int atomic_order_index = getAtomicOrderIndex(RMWI->getOrdering());
749 Value *val = RMWI->getValOperand();
750 Value *order = ConstantInt::get(OrdTy, atomic_order_index);
751 Value *args[] = {Addr, val, order, position};
752 Instruction* funcInst = CallInst::Create(CDSAtomicRMW[RMWI->getOperation()][Idx], args);
753 ReplaceInstWithInst(RMWI, funcInst);
754 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) {
755 IRBuilder<> IRB(CASI);
757 Value *Addr = CASI->getPointerOperand();
758 int Idx=getMemoryAccessFuncIndex(Addr, DL);
762 const unsigned ByteSize = 1U << Idx;
763 const unsigned BitSize = ByteSize * 8;
764 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize);
765 Type *PtrTy = Ty->getPointerTo();
767 Value *CmpOperand = IRB.CreateBitOrPointerCast(CASI->getCompareOperand(), Ty);
768 Value *NewOperand = IRB.CreateBitOrPointerCast(CASI->getNewValOperand(), Ty);
770 int atomic_order_index_succ = getAtomicOrderIndex(CASI->getSuccessOrdering());
771 int atomic_order_index_fail = getAtomicOrderIndex(CASI->getFailureOrdering());
772 Value *order_succ = ConstantInt::get(OrdTy, atomic_order_index_succ);
773 Value *order_fail = ConstantInt::get(OrdTy, atomic_order_index_fail);
775 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
776 CmpOperand, NewOperand,
777 order_succ, order_fail, position};
779 CallInst *funcInst = IRB.CreateCall(CDSAtomicCAS_V1[Idx], Args);
780 Value *Success = IRB.CreateICmpEQ(funcInst, CmpOperand);
782 Value *OldVal = funcInst;
783 Type *OrigOldValTy = CASI->getNewValOperand()->getType();
784 if (Ty != OrigOldValTy) {
785 // The value is a pointer, so we need to cast the return value.
786 OldVal = IRB.CreateIntToPtr(funcInst, OrigOldValTy);
790 IRB.CreateInsertValue(UndefValue::get(CASI->getType()), OldVal, 0);
791 Res = IRB.CreateInsertValue(Res, Success, 1);
793 I->replaceAllUsesWith(Res);
794 I->eraseFromParent();
795 } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) {
796 int atomic_order_index = getAtomicOrderIndex(FI->getOrdering());
797 Value *order = ConstantInt::get(OrdTy, atomic_order_index);
798 Value *Args[] = {order, position};
800 CallInst *funcInst = CallInst::Create(CDSAtomicThreadFence, Args);
801 ReplaceInstWithInst(FI, funcInst);
802 // errs() << "Thread Fences replaced\n";
807 bool CDSPass::isAtomicCall(Instruction *I) {
808 if ( auto *CI = dyn_cast<CallInst>(I) ) {
809 Function *fun = CI->getCalledFunction();
813 StringRef funName = fun->getName();
815 // TODO: come up with better rules for function name checking
816 for (StringRef name : AtomicFuncNames) {
817 if ( funName.contains(name) )
821 for (StringRef PartialName : PartialAtomicFuncNames) {
822 if (funName.contains(PartialName) &&
823 funName.contains("atomic") )
831 bool CDSPass::instrumentAtomicCall(CallInst *CI, const DataLayout &DL) {
833 Function *fun = CI->getCalledFunction();
834 StringRef funName = fun->getName();
835 std::vector<Value *> parameters;
837 User::op_iterator begin = CI->arg_begin();
838 User::op_iterator end = CI->arg_end();
839 for (User::op_iterator it = begin; it != end; ++it) {
841 parameters.push_back(param);
844 // obtain source line number of the CallInst
845 Value *position = getPosition(CI, IRB);
847 // the pointer to the address is always the first argument
848 Value *OrigPtr = parameters[0];
850 int Idx = getMemoryAccessFuncIndex(OrigPtr, DL);
854 const unsigned ByteSize = 1U << Idx;
855 const unsigned BitSize = ByteSize * 8;
856 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize);
857 Type *PtrTy = Ty->getPointerTo();
859 // atomic_init; args = {obj, order}
860 if (funName.contains("atomic_init")) {
861 Value *OrigVal = parameters[1];
863 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
865 if (OrigVal->getType()->isPtrOrPtrVectorTy())
866 val = IRB.CreatePointerCast(OrigVal, Ty);
868 val = IRB.CreateIntCast(OrigVal, Ty, true);
870 Value *args[] = {ptr, val, position};
872 Instruction* funcInst = CallInst::Create(CDSAtomicInit[Idx], args);
873 ReplaceInstWithInst(CI, funcInst);
878 // atomic_load; args = {obj, order}
879 if (funName.contains("atomic_load")) {
880 bool isExplicit = funName.contains("atomic_load_explicit");
882 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
885 order = IRB.CreateBitOrPointerCast(parameters[1], OrdTy);
887 order = ConstantInt::get(OrdTy,
888 (int) AtomicOrderingCABI::seq_cst);
889 Value *args[] = {ptr, order, position};
891 Instruction* funcInst = CallInst::Create(CDSAtomicLoad[Idx], args);
892 ReplaceInstWithInst(CI, funcInst);
895 } else if (funName.contains("atomic") &&
896 funName.contains("load") ) {
897 // does this version of call always have an atomic order as an argument?
898 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
899 Value *order = IRB.CreateBitOrPointerCast(parameters[1], OrdTy);
900 Value *args[] = {ptr, order, position};
902 if (!CI->getType()->isPointerTy()) {
906 CallInst *funcInst = IRB.CreateCall(CDSAtomicLoad[Idx], args);
907 Value *RetVal = IRB.CreateIntToPtr(funcInst, CI->getType());
909 CI->replaceAllUsesWith(RetVal);
910 CI->eraseFromParent();
915 // atomic_store; args = {obj, val, order}
916 if (funName.contains("atomic_store")) {
917 bool isExplicit = funName.contains("atomic_store_explicit");
918 Value *OrigVal = parameters[1];
920 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
921 Value *val = IRB.CreatePointerCast(OrigVal, Ty);
924 order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
926 order = ConstantInt::get(OrdTy,
927 (int) AtomicOrderingCABI::seq_cst);
928 Value *args[] = {ptr, val, order, position};
930 Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
931 ReplaceInstWithInst(CI, funcInst);
934 } else if (funName.contains("atomic") &&
935 funName.contains("store") ) {
936 // Does this version of call always have an atomic order as an argument?
937 Value *OrigVal = parameters[1];
939 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
941 if (OrigVal->getType()->isPtrOrPtrVectorTy())
942 val = IRB.CreatePointerCast(OrigVal, Ty);
944 val = IRB.CreateIntCast(OrigVal, Ty, true);
946 Value *order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
947 Value *args[] = {ptr, val, order, position};
949 Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
950 ReplaceInstWithInst(CI, funcInst);
955 // atomic_fetch_*; args = {obj, val, order}
956 if (funName.contains("atomic_fetch_") ||
957 funName.contains("atomic_exchange")) {
959 /* TODO: implement stricter function name checking */
960 if (funName.contains("non"))
963 bool isExplicit = funName.contains("_explicit");
964 Value *OrigVal = parameters[1];
967 if ( funName.contains("_fetch_add") )
968 op = AtomicRMWInst::Add;
969 else if ( funName.contains("_fetch_sub") )
970 op = AtomicRMWInst::Sub;
971 else if ( funName.contains("_fetch_and") )
972 op = AtomicRMWInst::And;
973 else if ( funName.contains("_fetch_or") )
974 op = AtomicRMWInst::Or;
975 else if ( funName.contains("_fetch_xor") )
976 op = AtomicRMWInst::Xor;
977 else if ( funName.contains("atomic_exchange") )
978 op = AtomicRMWInst::Xchg;
980 errs() << "Unknown atomic read-modify-write operation\n";
984 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
986 if (OrigVal->getType()->isPtrOrPtrVectorTy())
987 val = IRB.CreatePointerCast(OrigVal, Ty);
989 val = IRB.CreateIntCast(OrigVal, Ty, true);
993 order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
995 order = ConstantInt::get(OrdTy,
996 (int) AtomicOrderingCABI::seq_cst);
997 Value *args[] = {ptr, val, order, position};
999 Instruction* funcInst = CallInst::Create(CDSAtomicRMW[op][Idx], args);
1000 ReplaceInstWithInst(CI, funcInst);
1003 } else if (funName.contains("fetch")) {
1004 errs() << "atomic fetch captured. Not implemented yet. ";
1005 errs() << "See source file :";
1006 getPosition(CI, IRB, true);
1008 } else if (funName.contains("exchange") &&
1009 !funName.contains("compare_exchange") ) {
1010 if (CI->getType()->isPointerTy()) {
1012 * TODO: instrument the following case
1014 * std::atomic<struct T *> m_tail;
1016 * struct T * pred = m_tail.exchange(me, memory_order_*);
1018 errs() << "atomic exchange captured. Not implemented yet. ";
1019 errs() << "See source file :";
1020 getPosition(CI, IRB, true);
1025 Value *OrigVal = parameters[1];
1027 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
1029 if (OrigVal->getType()->isPtrOrPtrVectorTy())
1030 val = IRB.CreatePointerCast(OrigVal, Ty);
1032 val = IRB.CreateIntCast(OrigVal, Ty, true);
1034 Value *order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
1035 Value *args[] = {ptr, val, order, position};
1036 int op = AtomicRMWInst::Xchg;
1038 Instruction* funcInst = CallInst::Create(CDSAtomicRMW[op][Idx], args);
1039 ReplaceInstWithInst(CI, funcInst);
1044 /* atomic_compare_exchange_*;
1045 args = {obj, expected, new value, order1, order2}
1047 if ( funName.contains("atomic_compare_exchange_") ) {
1048 bool isExplicit = funName.contains("_explicit");
1050 Value *Addr = IRB.CreatePointerCast(OrigPtr, PtrTy);
1051 Value *CmpOperand = IRB.CreatePointerCast(parameters[1], PtrTy);
1052 Value *NewOperand = IRB.CreateBitOrPointerCast(parameters[2], Ty);
1054 Value *order_succ, *order_fail;
1056 order_succ = IRB.CreateBitOrPointerCast(parameters[3], OrdTy);
1058 if (parameters.size() > 4) {
1059 order_fail = IRB.CreateBitOrPointerCast(parameters[4], OrdTy);
1061 /* The failure order is not provided */
1062 order_fail = order_succ;
1063 ConstantInt * order_succ_cast = dyn_cast<ConstantInt>(order_succ);
1064 int index = order_succ_cast->getSExtValue();
1066 order_fail = ConstantInt::get(OrdTy,
1067 AtomicCasFailureOrderIndex(index));
1070 order_succ = ConstantInt::get(OrdTy,
1071 (int) AtomicOrderingCABI::seq_cst);
1072 order_fail = ConstantInt::get(OrdTy,
1073 (int) AtomicOrderingCABI::seq_cst);
1076 Value *args[] = {Addr, CmpOperand, NewOperand,
1077 order_succ, order_fail, position};
1079 Instruction* funcInst = CallInst::Create(CDSAtomicCAS_V2[Idx], args);
1080 ReplaceInstWithInst(CI, funcInst);
1083 } else if ( funName.contains("compare_exchange_strong") ||
1084 funName.contains("compare_exchange_weak") ) {
1085 Value *Addr = IRB.CreatePointerCast(OrigPtr, PtrTy);
1086 Value *CmpOperand = IRB.CreatePointerCast(parameters[1], PtrTy);
1087 Value *NewOperand = IRB.CreateBitOrPointerCast(parameters[2], Ty);
1089 Value *order_succ, *order_fail;
1090 order_succ = IRB.CreateBitOrPointerCast(parameters[3], OrdTy);
1092 if (parameters.size() > 4) {
1093 order_fail = IRB.CreateBitOrPointerCast(parameters[4], OrdTy);
1095 /* The failure order is not provided */
1096 order_fail = order_succ;
1097 ConstantInt * order_succ_cast = dyn_cast<ConstantInt>(order_succ);
1098 int index = order_succ_cast->getSExtValue();
1100 order_fail = ConstantInt::get(OrdTy,
1101 AtomicCasFailureOrderIndex(index));
1104 Value *args[] = {Addr, CmpOperand, NewOperand,
1105 order_succ, order_fail, position};
1106 Instruction* funcInst = CallInst::Create(CDSAtomicCAS_V2[Idx], args);
1107 ReplaceInstWithInst(CI, funcInst);
1115 int CDSPass::getMemoryAccessFuncIndex(Value *Addr,
1116 const DataLayout &DL) {
1117 Type *OrigPtrTy = Addr->getType();
1118 Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType();
1119 assert(OrigTy->isSized());
1120 uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy);
1121 if (TypeSize != 8 && TypeSize != 16 &&
1122 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) {
1123 NumAccessesWithBadSize++;
1124 // Ignore all unusual sizes.
1127 size_t Idx = countTrailingZeros(TypeSize / 8);
1128 //assert(Idx < kNumberOfAccessSizes);
1129 if (Idx >= kNumberOfAccessSizes) {
1136 char CDSPass::ID = 0;
1138 // Automatically enable the pass.
1139 static void registerCDSPass(const PassManagerBuilder &,
1140 legacy::PassManagerBase &PM) {
1141 PM.add(new CDSPass());
1144 /* Enable the pass when opt level is greater than 0 */
1145 static RegisterStandardPasses
1146 RegisterMyPass1(PassManagerBuilder::EP_OptimizerLast,
1149 /* Enable the pass when opt level is 0 */
1150 static RegisterStandardPasses
1151 RegisterMyPass2(PassManagerBuilder::EP_EnabledOnOptLevel0,