CDSPass.cpp

   1 //===-- CDSPass.cpp - xxx -------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 // This file is distributed under the University of Illinois Open Source
   7 // License. See LICENSE.TXT for details.
   8 //
   9 //===----------------------------------------------------------------------===//
  10 //
  11 // This file is a modified version of ThreadSanitizer.cpp, a part of a race detector.
  12 //
  13 // The tool is under development, for the details about previous versions see
  14 // http://code.google.com/p/data-race-test
  15 //
  16 // The instrumentation phase is quite simple:
  17 //   - Insert calls to run-time library before every memory access.
  18 //      - Optimizations may apply to avoid instrumenting some of the accesses.
  19 //   - Insert calls at function entry/exit.
  20 // The rest is handled by the run-time library.
  21 //===----------------------------------------------------------------------===//
  22
  23 #include "llvm/ADT/Statistic.h"
  24 #include "llvm/ADT/StringExtras.h"
  25 #include "llvm/ADT/SmallString.h"
  26 #include "llvm/Analysis/ValueTracking.h"
  27 #include "llvm/Analysis/CaptureTracking.h"
  28 #include "llvm/IR/BasicBlock.h"
  29 #include "llvm/IR/Function.h"
  30 #include "llvm/IR/IRBuilder.h"
  31 #include "llvm/IR/Instructions.h"
  32 #include "llvm/IR/IntrinsicInst.h"
  33 #include "llvm/IR/LLVMContext.h"
  34 #include "llvm/IR/LegacyPassManager.h"
  35 #include "llvm/IR/Module.h"
  36 #include "llvm/IR/PassManager.h"
  37 #include "llvm/Pass.h"
  38 #include "llvm/ProfileData/InstrProf.h"
  39 #include "llvm/Support/raw_ostream.h"
  40 #include "llvm/Support/AtomicOrdering.h"
  41 #include "llvm/Support/Debug.h"
  42 #include "llvm/Transforms/Scalar.h"
  43 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
  44 #include "llvm/Transforms/Utils/EscapeEnumerator.h"
  45 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
  46 #include <vector>
  47
  48 using namespace llvm;
  49
  50 #define DEBUG_TYPE "CDS"
  51 #include <llvm/IR/DebugLoc.h>
  52
  53 Value *getPosition( Instruction * I, IRBuilder <> IRB, bool print = false)
  54 {
  55         const DebugLoc & debug_location = I->getDebugLoc ();
  56         std::string position_string;
  57         {
  58                 llvm::raw_string_ostream position_stream (position_string);
  59                 debug_location . print (position_stream);
  60         }
  61
  62         if (print) {
  63                 errs() << position_string << "\n";
  64         }
  65
  66         return IRB.CreateGlobalStringPtr (position_string);
  67 }
  68
  69 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
  70 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
  71 STATISTIC(NumOmittedReadsBeforeWrite,
  72           "Number of reads ignored due to following writes");
  73 STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size");
  74 // STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes");
  75 // STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads");
  76 STATISTIC(NumOmittedReadsFromConstantGlobals,
  77           "Number of reads from constant globals");
  78 STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads");
  79 STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing");
  80
  81 // static const char *const kCDSModuleCtorName = "cds.module_ctor";
  82 // static const char *const kCDSInitName = "cds_init";
  83
  84 Type * OrdTy;
  85 Type * IntPtrTy;
  86 Type * Int8PtrTy;
  87 Type * Int16PtrTy;
  88 Type * Int32PtrTy;
  89 Type * Int64PtrTy;
  90
  91 Type * VoidTy;
  92
  93 static const size_t kNumberOfAccessSizes = 4;
  94
  95 int getAtomicOrderIndex(AtomicOrdering order) {
  96         switch (order) {
  97                 case AtomicOrdering::Monotonic:
  98                         return (int)AtomicOrderingCABI::relaxed;
  99                 //  case AtomicOrdering::Consume:         // not specified yet
 100                 //    return AtomicOrderingCABI::consume;
 101                 case AtomicOrdering::Acquire:
 102                         return (int)AtomicOrderingCABI::acquire;
 103                 case AtomicOrdering::Release:
 104                         return (int)AtomicOrderingCABI::release;
 105                 case AtomicOrdering::AcquireRelease:
 106                         return (int)AtomicOrderingCABI::acq_rel;
 107                 case AtomicOrdering::SequentiallyConsistent:
 108                         return (int)AtomicOrderingCABI::seq_cst;
 109                 default:
 110                         // unordered or Not Atomic
 111                         return -1;
 112         }
 113 }
 114
 115 AtomicOrderingCABI indexToAtomicOrder(int index) {
 116         switch (index) {
 117                 case 0:
 118                         return AtomicOrderingCABI::relaxed;
 119                 case 1:
 120                         return AtomicOrderingCABI::consume;
 121                 case 2:
 122                         return AtomicOrderingCABI::acquire;
 123                 case 3:
 124                         return AtomicOrderingCABI::release;
 125                 case 4:
 126                         return AtomicOrderingCABI::acq_rel;
 127                 case 5:
 128                         return AtomicOrderingCABI::seq_cst;
 129                 default:
 130                         errs() << "Bad Atomic index\n";
 131                         return AtomicOrderingCABI::seq_cst;
 132         }
 133 }
 134
 135 /* According to atomic_base.h: __cmpexch_failure_order */
 136 int AtomicCasFailureOrderIndex(int index) {
 137         AtomicOrderingCABI succ_order = indexToAtomicOrder(index);
 138         AtomicOrderingCABI fail_order;
 139         if (succ_order == AtomicOrderingCABI::acq_rel)
 140                 fail_order = AtomicOrderingCABI::acquire;
 141         else if (succ_order == AtomicOrderingCABI::release)
 142                 fail_order = AtomicOrderingCABI::relaxed;
 143         else
 144                 fail_order = succ_order;
 145
 146         return (int) fail_order;
 147 }
 148
 149 /* The original function checkSanitizerInterfaceFunction was defined
 150  * in llvm/Transforms/Utils/ModuleUtils.h
 151  */
 152 static Function * checkCDSPassInterfaceFunction(Constant *FuncOrBitcast) {
 153         if (isa<Function>(FuncOrBitcast))
 154                 return cast<Function>(FuncOrBitcast);
 155         FuncOrBitcast->print(errs());
 156         errs() << '\n';
 157         std::string Err;
 158         raw_string_ostream Stream(Err);
 159         Stream << "CDSPass interface function redefined: " << *FuncOrBitcast;
 160         report_fatal_error(Err);
 161 }
 162
 163 namespace {
 164         struct CDSPass : public FunctionPass {
 165                 CDSPass() : FunctionPass(ID) {}
 166                 StringRef getPassName() const override;
 167                 bool runOnFunction(Function &F) override;
 168                 bool doInitialization(Module &M) override;
 169                 static char ID;
 170
 171         private:
 172                 void initializeCallbacks(Module &M);
 173                 bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL);
 174                 bool instrumentVolatile(Instruction *I, const DataLayout &DL);
 175                 bool instrumentMemIntrinsic(Instruction *I);
 176                 bool isAtomicCall(Instruction *I);
 177                 bool instrumentAtomic(Instruction *I, const DataLayout &DL);
 178                 bool instrumentAtomicCall(CallInst *CI, const DataLayout &DL);
 179                 bool shouldInstrumentBeforeAtomics(Instruction *I);
 180                 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local,
 181                                                                                         SmallVectorImpl<Instruction *> &All,
 182                                                                                         const DataLayout &DL);
 183                 bool addrPointsToConstantData(Value *Addr);
 184                 int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL);
 185
 186                 Function * CDSFuncEntry;
 187                 Function * CDSFuncExit;
 188
 189                 Function * CDSLoad[kNumberOfAccessSizes];
 190                 Function * CDSStore[kNumberOfAccessSizes];
 191                 Function * CDSVolatileLoad[kNumberOfAccessSizes];
 192                 Function * CDSVolatileStore[kNumberOfAccessSizes];
 193                 Function * CDSAtomicInit[kNumberOfAccessSizes];
 194                 Function * CDSAtomicLoad[kNumberOfAccessSizes];
 195                 Function * CDSAtomicStore[kNumberOfAccessSizes];
 196                 Function * CDSAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes];
 197                 Function * CDSAtomicCAS_V1[kNumberOfAccessSizes];
 198                 Function * CDSAtomicCAS_V2[kNumberOfAccessSizes];
 199                 Function * CDSAtomicThreadFence;
 200                 Function * MemmoveFn, * MemcpyFn, * MemsetFn;
 201                 // Function * CDSCtorFunction;
 202
 203                 std::vector<StringRef> AtomicFuncNames;
 204                 std::vector<StringRef> PartialAtomicFuncNames;
 205         };
 206 }
 207
 208 StringRef CDSPass::getPassName() const {
 209         return "CDSPass";
 210 }
 211
 212 void CDSPass::initializeCallbacks(Module &M) {
 213         LLVMContext &Ctx = M.getContext();
 214         AttributeList Attr;
 215         Attr = Attr.addAttribute(Ctx, AttributeList::FunctionIndex,
 216                         Attribute::NoUnwind);
 217
 218         Type * Int1Ty = Type::getInt1Ty(Ctx);
 219         Type * Int32Ty = Type::getInt32Ty(Ctx);
 220         OrdTy = Type::getInt32Ty(Ctx);
 221
 222         Int8PtrTy  = Type::getInt8PtrTy(Ctx);
 223         Int16PtrTy = Type::getInt16PtrTy(Ctx);
 224         Int32PtrTy = Type::getInt32PtrTy(Ctx);
 225         Int64PtrTy = Type::getInt64PtrTy(Ctx);
 226
 227         VoidTy = Type::getVoidTy(Ctx);
 228
 229         CDSFuncEntry = checkCDSPassInterfaceFunction(
 230                                                 M.getOrInsertFunction("cds_func_entry",
 231                                                 Attr, VoidTy, Int8PtrTy));
 232         CDSFuncExit = checkCDSPassInterfaceFunction(
 233                                                 M.getOrInsertFunction("cds_func_exit",
 234                                                 Attr, VoidTy, Int8PtrTy));
 235
 236         // Get the function to call from our untime library.
 237         for (unsigned i = 0; i < kNumberOfAccessSizes; i++) {
 238                 const unsigned ByteSize = 1U << i;
 239                 const unsigned BitSize = ByteSize * 8;
 240
 241                 std::string ByteSizeStr = utostr(ByteSize);
 242                 std::string BitSizeStr = utostr(BitSize);
 243
 244                 Type *Ty = Type::getIntNTy(Ctx, BitSize);
 245                 Type *PtrTy = Ty->getPointerTo();
 246
 247                 // uint8_t cds_atomic_load8 (void * obj, int atomic_index)
 248                 // void cds_atomic_store8 (void * obj, int atomic_index, uint8_t val)
 249                 SmallString<32> LoadName("cds_load" + BitSizeStr);
 250                 SmallString<32> StoreName("cds_store" + BitSizeStr);
 251                 SmallString<32> VolatileLoadName("cds_volatile_load" + BitSizeStr);
 252                 SmallString<32> VolatileStoreName("cds_volatile_store" + BitSizeStr);
 253                 SmallString<32> AtomicInitName("cds_atomic_init" + BitSizeStr);
 254                 SmallString<32> AtomicLoadName("cds_atomic_load" + BitSizeStr);
 255                 SmallString<32> AtomicStoreName("cds_atomic_store" + BitSizeStr);
 256
 257                 CDSLoad[i]  = checkCDSPassInterfaceFunction(
 258                                                         M.getOrInsertFunction(LoadName, Attr, VoidTy, PtrTy));
 259                 CDSStore[i] = checkCDSPassInterfaceFunction(
 260                                                         M.getOrInsertFunction(StoreName, Attr, VoidTy, PtrTy));
 261                 CDSVolatileLoad[i]  = checkCDSPassInterfaceFunction(
 262                                                                 M.getOrInsertFunction(VolatileLoadName,
 263                                                                 Attr, Ty, PtrTy, Int8PtrTy));
 264                 CDSVolatileStore[i] = checkCDSPassInterfaceFunction(
 265                                                                 M.getOrInsertFunction(VolatileStoreName,
 266                                                                 Attr, VoidTy, PtrTy, Ty, Int8PtrTy));
 267                 CDSAtomicInit[i] = checkCDSPassInterfaceFunction(
 268                                                         M.getOrInsertFunction(AtomicInitName,
 269                                                         Attr, VoidTy, PtrTy, Ty, Int8PtrTy));
 270                 CDSAtomicLoad[i]  = checkCDSPassInterfaceFunction(
 271                                                                 M.getOrInsertFunction(AtomicLoadName,
 272                                                                 Attr, Ty, PtrTy, OrdTy, Int8PtrTy));
 273                 CDSAtomicStore[i] = checkCDSPassInterfaceFunction(
 274                                                                 M.getOrInsertFunction(AtomicStoreName,
 275                                                                 Attr, VoidTy, PtrTy, Ty, OrdTy, Int8PtrTy));
 276
 277                 for (int op = AtomicRMWInst::FIRST_BINOP;
 278                         op <= AtomicRMWInst::LAST_BINOP; ++op) {
 279                         CDSAtomicRMW[op][i] = nullptr;
 280                         std::string NamePart;
 281
 282                         if (op == AtomicRMWInst::Xchg)
 283                                 NamePart = "_exchange";
 284                         else if (op == AtomicRMWInst::Add)
 285                                 NamePart = "_fetch_add";
 286                         else if (op == AtomicRMWInst::Sub)
 287                                 NamePart = "_fetch_sub";
 288                         else if (op == AtomicRMWInst::And)
 289                                 NamePart = "_fetch_and";
 290                         else if (op == AtomicRMWInst::Or)
 291                                 NamePart = "_fetch_or";
 292                         else if (op == AtomicRMWInst::Xor)
 293                                 NamePart = "_fetch_xor";
 294                         else
 295                                 continue;
 296
 297                         SmallString<32> AtomicRMWName("cds_atomic" + NamePart + BitSizeStr);
 298                         CDSAtomicRMW[op][i] = checkCDSPassInterfaceFunction(
 299                                                                         M.getOrInsertFunction(AtomicRMWName,
 300                                                                         Attr, Ty, PtrTy, Ty, OrdTy, Int8PtrTy));
 301                 }
 302
 303                 // only supportes strong version
 304                 SmallString<32> AtomicCASName_V1("cds_atomic_compare_exchange" + BitSizeStr + "_v1");
 305                 SmallString<32> AtomicCASName_V2("cds_atomic_compare_exchange" + BitSizeStr + "_v2");
 306                 CDSAtomicCAS_V1[i] = checkCDSPassInterfaceFunction(
 307                                                                 M.getOrInsertFunction(AtomicCASName_V1,
 308                                                                 Attr, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, Int8PtrTy));
 309                 CDSAtomicCAS_V2[i] = checkCDSPassInterfaceFunction(
 310                                                                 M.getOrInsertFunction(AtomicCASName_V2,
 311                                                                 Attr, Int1Ty, PtrTy, PtrTy, Ty, OrdTy, OrdTy, Int8PtrTy));
 312         }
 313
 314         CDSAtomicThreadFence = checkCDSPassInterfaceFunction(
 315                         M.getOrInsertFunction("cds_atomic_thread_fence", Attr, VoidTy, OrdTy, Int8PtrTy));
 316
 317         MemmoveFn = checkCDSPassInterfaceFunction(
 318                                         M.getOrInsertFunction("memmove", Attr, Int8PtrTy, Int8PtrTy,
 319                                         Int8PtrTy, IntPtrTy));
 320         MemcpyFn = checkCDSPassInterfaceFunction(
 321                                         M.getOrInsertFunction("memcpy", Attr, Int8PtrTy, Int8PtrTy,
 322                                         Int8PtrTy, IntPtrTy));
 323         MemsetFn = checkCDSPassInterfaceFunction(
 324                                         M.getOrInsertFunction("memset", Attr, Int8PtrTy, Int8PtrTy,
 325                                         Int32Ty, IntPtrTy));
 326 }
 327
 328 bool CDSPass::doInitialization(Module &M) {
 329         const DataLayout &DL = M.getDataLayout();
 330         IntPtrTy = DL.getIntPtrType(M.getContext());
 331
 332         // createSanitizerCtorAndInitFunctions is defined in "llvm/Transforms/Utils/ModuleUtils.h"
 333         // We do not support it yet
 334         /*
 335         std::tie(CDSCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions(
 336                         M, kCDSModuleCtorName, kCDSInitName, {}, {});
 337
 338         appendToGlobalCtors(M, CDSCtorFunction, 0);
 339         */
 340
 341         AtomicFuncNames =
 342         {
 343                 "atomic_init", "atomic_load", "atomic_store",
 344                 "atomic_fetch_", "atomic_exchange", "atomic_compare_exchange_"
 345         };
 346
 347         PartialAtomicFuncNames =
 348         {
 349                 "load", "store", "fetch", "exchange", "compare_exchange_"
 350         };
 351
 352         return true;
 353 }
 354
 355 static bool isVtableAccess(Instruction *I) {
 356         if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa))
 357                 return Tag->isTBAAVtableAccess();
 358         return false;
 359 }
 360
 361 // Do not instrument known races/"benign races" that come from compiler
 362 // instrumentatin. The user has no way of suppressing them.
 363 static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) {
 364         // Peel off GEPs and BitCasts.
 365         Addr = Addr->stripInBoundsOffsets();
 366
 367         if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
 368                 if (GV->hasSection()) {
 369                         StringRef SectionName = GV->getSection();
 370                         // Check if the global is in the PGO counters section.
 371                         auto OF = Triple(M->getTargetTriple()).getObjectFormat();
 372                         if (SectionName.endswith(
 373                               getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
 374                                 return false;
 375                 }
 376
 377                 // Check if the global is private gcov data.
 378                 if (GV->getName().startswith("__llvm_gcov") ||
 379                 GV->getName().startswith("__llvm_gcda"))
 380                 return false;
 381         }
 382
 383         // Do not instrument acesses from different address spaces; we cannot deal
 384         // with them.
 385         if (Addr) {
 386                 Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType());
 387                 if (PtrTy->getPointerAddressSpace() != 0)
 388                         return false;
 389         }
 390
 391         return true;
 392 }
 393
 394 bool CDSPass::addrPointsToConstantData(Value *Addr) {
 395         // If this is a GEP, just analyze its pointer operand.
 396         if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr))
 397                 Addr = GEP->getPointerOperand();
 398
 399         if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
 400                 if (GV->isConstant()) {
 401                         // Reads from constant globals can not race with any writes.
 402                         NumOmittedReadsFromConstantGlobals++;
 403                         return true;
 404                 }
 405         } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) {
 406                 if (isVtableAccess(L)) {
 407                         // Reads from a vtable pointer can not race with any writes.
 408                         NumOmittedReadsFromVtable++;
 409                         return true;
 410                 }
 411         }
 412         return false;
 413 }
 414
 415 bool CDSPass::shouldInstrumentBeforeAtomics(Instruction * Inst) {
 416         if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
 417                 AtomicOrdering ordering = LI->getOrdering();
 418                 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
 419                         return true;
 420         } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
 421                 AtomicOrdering ordering = SI->getOrdering();
 422                 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
 423                         return true;
 424         } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst)) {
 425                 AtomicOrdering ordering = RMWI->getOrdering();
 426                 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
 427                         return true;
 428         } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(Inst)) {
 429                 AtomicOrdering ordering = CASI->getSuccessOrdering();
 430                 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
 431                         return true;
 432         } else if (FenceInst *FI = dyn_cast<FenceInst>(Inst)) {
 433                 AtomicOrdering ordering = FI->getOrdering();
 434                 if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
 435                         return true;
 436         }
 437
 438         return false;
 439 }
 440
 441 void CDSPass::chooseInstructionsToInstrument(
 442         SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All,
 443         const DataLayout &DL) {
 444         SmallPtrSet<Value*, 8> WriteTargets;
 445         // Iterate from the end.
 446         for (Instruction *I : reverse(Local)) {
 447                 if (StoreInst *Store = dyn_cast<StoreInst>(I)) {
 448                         Value *Addr = Store->getPointerOperand();
 449                         if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr))
 450                                 continue;
 451                         WriteTargets.insert(Addr);
 452                 } else {
 453                         LoadInst *Load = cast<LoadInst>(I);
 454                         Value *Addr = Load->getPointerOperand();
 455                         if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr))
 456                                 continue;
 457                         if (WriteTargets.count(Addr)) {
 458                                 // We will write to this temp, so no reason to analyze the read.
 459                                 NumOmittedReadsBeforeWrite++;
 460                                 continue;
 461                         }
 462                         if (addrPointsToConstantData(Addr)) {
 463                                 // Addr points to some constant data -- it can not race with any writes.
 464                                 continue;
 465                         }
 466                 }
 467                 Value *Addr = isa<StoreInst>(*I)
 468                         ? cast<StoreInst>(I)->getPointerOperand()
 469                         : cast<LoadInst>(I)->getPointerOperand();
 470                 if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) &&
 471                                 !PointerMayBeCaptured(Addr, true, true)) {
 472                         // The variable is addressable but not captured, so it cannot be
 473                         // referenced from a different thread and participate in a data race
 474                         // (see llvm/Analysis/CaptureTracking.h for details).
 475                         NumOmittedNonCaptured++;
 476                         continue;
 477                 }
 478                 All.push_back(I);
 479         }
 480         Local.clear();
 481 }
 482
 483 /* Not implemented
 484 void CDSPass::InsertRuntimeIgnores(Function &F) {
 485         IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
 486         IRB.CreateCall(CDSIgnoreBegin);
 487         EscapeEnumerator EE(F, "cds_ignore_cleanup", ClHandleCxxExceptions);
 488         while (IRBuilder<> *AtExit = EE.Next()) {
 489                 AtExit->CreateCall(CDSIgnoreEnd);
 490         }
 491 }*/
 492
 493 bool CDSPass::runOnFunction(Function &F) {
 494         if (F.getName() == "main") {
 495                 F.setName("user_main");
 496                 errs() << "main replaced by user_main\n";
 497         }
 498
 499         initializeCallbacks( *F.getParent() );
 500         SmallVector<Instruction*, 8> AllLoadsAndStores;
 501         SmallVector<Instruction*, 8> LocalLoadsAndStores;
 502         SmallVector<Instruction*, 8> VolatileLoadsAndStores;
 503         SmallVector<Instruction*, 8> AtomicAccesses;
 504         SmallVector<Instruction*, 8> MemIntrinCalls;
 505
 506         bool Res = false;
 507         bool HasAtomic = false;
 508         bool HasVolatile = false;
 509         const DataLayout &DL = F.getParent()->getDataLayout();
 510
 511         for (auto &BB : F) {
 512                 for (auto &Inst : BB) {
 513                         if ( (&Inst)->isAtomic() ) {
 514                                 AtomicAccesses.push_back(&Inst);
 515                                 HasAtomic = true;
 516
 517                                 if (shouldInstrumentBeforeAtomics(&Inst)) {
 518                                         chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores,
 519                                                 DL);
 520                                 }
 521                         } else if (isAtomicCall(&Inst) ) {
 522                                 AtomicAccesses.push_back(&Inst);
 523                                 HasAtomic = true;
 524                                 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores,
 525                                         DL);
 526                         } else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) {
 527                                 LoadInst *LI = dyn_cast<LoadInst>(&Inst);
 528                                 StoreInst *SI = dyn_cast<StoreInst>(&Inst);
 529                                 bool isVolatile = ( LI ? LI->isVolatile() : SI->isVolatile() );
 530
 531                                 if (isVolatile) {
 532                                         VolatileLoadsAndStores.push_back(&Inst);
 533                                         HasVolatile = true;
 534                                 } else
 535                                         LocalLoadsAndStores.push_back(&Inst);
 536                         } else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) {
 537                                 if (isa<MemIntrinsic>(Inst))
 538                                         MemIntrinCalls.push_back(&Inst);
 539
 540                                 /*if (CallInst *CI = dyn_cast<CallInst>(&Inst))
 541                                         maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI);
 542                                 */
 543
 544                                 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores,
 545                                         DL);
 546                         }
 547                 }
 548
 549                 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL);
 550         }
 551
 552         for (auto Inst : AllLoadsAndStores) {
 553                 Res |= instrumentLoadOrStore(Inst, DL);
 554         }
 555
 556         for (auto Inst : VolatileLoadsAndStores) {
 557                 Res |= instrumentVolatile(Inst, DL);
 558         }
 559
 560         for (auto Inst : AtomicAccesses) {
 561                 Res |= instrumentAtomic(Inst, DL);
 562         }
 563
 564         for (auto Inst : MemIntrinCalls) {
 565                 Res |= instrumentMemIntrinsic(Inst);
 566         }
 567
 568         // Only instrument functions that contain atomics or volatiles
 569         if (Res && ( HasAtomic || HasVolatile) ) {
 570                 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
 571                 /* Unused for now
 572                 Value *ReturnAddress = IRB.CreateCall(
 573                         Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress),
 574                         IRB.getInt32(0));
 575                 */
 576
 577                 Value * FuncName = IRB.CreateGlobalStringPtr(F.getName());
 578                 IRB.CreateCall(CDSFuncEntry, FuncName);
 579
 580                 EscapeEnumerator EE(F, "cds_cleanup", true);
 581                 while (IRBuilder<> *AtExit = EE.Next()) {
 582                   AtExit->CreateCall(CDSFuncExit, FuncName);
 583                 }
 584
 585                 Res = true;
 586         }
 587
 588         return false;
 589 }
 590
 591 bool CDSPass::instrumentLoadOrStore(Instruction *I,
 592                                                                         const DataLayout &DL) {
 593         IRBuilder<> IRB(I);
 594         bool IsWrite = isa<StoreInst>(*I);
 595         Value *Addr = IsWrite
 596                 ? cast<StoreInst>(I)->getPointerOperand()
 597                 : cast<LoadInst>(I)->getPointerOperand();
 598
 599         // swifterror memory addresses are mem2reg promoted by instruction selection.
 600         // As such they cannot have regular uses like an instrumentation function and
 601         // it makes no sense to track them as memory.
 602         if (Addr->isSwiftError())
 603                 return false;
 604
 605         int Idx = getMemoryAccessFuncIndex(Addr, DL);
 606         if (Idx < 0)
 607                 return false;
 608
 609         if (IsWrite && isVtableAccess(I)) {
 610                 /* TODO
 611                 LLVM_DEBUG(dbgs() << "  VPTR : " << *I << "\n");
 612                 Value *StoredValue = cast<StoreInst>(I)->getValueOperand();
 613                 // StoredValue may be a vector type if we are storing several vptrs at once.
 614                 // In this case, just take the first element of the vector since this is
 615                 // enough to find vptr races.
 616                 if (isa<VectorType>(StoredValue->getType()))
 617                         StoredValue = IRB.CreateExtractElement(
 618                                         StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0));
 619                 if (StoredValue->getType()->isIntegerTy())
 620                         StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy());
 621                 // Call TsanVptrUpdate.
 622                 IRB.CreateCall(TsanVptrUpdate,
 623                                                 {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
 624                                                         IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())});
 625                 NumInstrumentedVtableWrites++;
 626                 */
 627                 return true;
 628         }
 629
 630         if (!IsWrite && isVtableAccess(I)) {
 631                 /* TODO
 632                 IRB.CreateCall(TsanVptrLoad,
 633                                                  IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()));
 634                 NumInstrumentedVtableReads++;
 635                 */
 636                 return true;
 637         }
 638
 639         // TODO: unaligned reads and writes
 640
 641         Value *OnAccessFunc = nullptr;
 642         OnAccessFunc = IsWrite ? CDSStore[Idx] : CDSLoad[Idx];
 643
 644         Type *ArgType = IRB.CreatePointerCast(Addr, Addr->getType())->getType();
 645
 646         if ( ArgType != Int8PtrTy && ArgType != Int16PtrTy &&
 647                         ArgType != Int32PtrTy && ArgType != Int64PtrTy ) {
 648                 // if other types of load or stores are passed in
 649                 return false;
 650         }
 651
 652         IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, Addr->getType()));
 653         if (IsWrite) NumInstrumentedWrites++;
 654         else         NumInstrumentedReads++;
 655         return true;
 656 }
 657
 658 bool CDSPass::instrumentVolatile(Instruction * I, const DataLayout &DL) {
 659         IRBuilder<> IRB(I);
 660         Value *position = getPosition(I, IRB);
 661
 662         if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
 663                 assert( LI->isVolatile() );
 664                 Value *Addr = LI->getPointerOperand();
 665                 int Idx=getMemoryAccessFuncIndex(Addr, DL);
 666                 if (Idx < 0)
 667                         return false;
 668
 669                 Value *args[] = {Addr, position};
 670                 Instruction* funcInst = CallInst::Create(CDSVolatileLoad[Idx], args);
 671                 ReplaceInstWithInst(LI, funcInst);
 672         } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
 673                 assert( SI->isVolatile() );
 674                 Value *Addr = SI->getPointerOperand();
 675                 int Idx=getMemoryAccessFuncIndex(Addr, DL);
 676                 if (Idx < 0)
 677                         return false;
 678
 679                 Value *val = SI->getValueOperand();
 680                 Value *args[] = {Addr, val, position};
 681                 Instruction* funcInst = CallInst::Create(CDSVolatileStore[Idx], args);
 682                 ReplaceInstWithInst(SI, funcInst);
 683         } else {
 684                 return false;
 685         }
 686
 687         return true;
 688 }
 689
 690 bool CDSPass::instrumentMemIntrinsic(Instruction *I) {
 691         IRBuilder<> IRB(I);
 692         if (MemSetInst *M = dyn_cast<MemSetInst>(I)) {
 693                 IRB.CreateCall(
 694                         MemsetFn,
 695                         {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()),
 696                          IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false),
 697                          IRB.CreateIntCast(M->getArgOperand(2), IntPtrTy, false)});
 698                 I->eraseFromParent();
 699         } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) {
 700                 IRB.CreateCall(
 701                         isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn,
 702                         {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()),
 703                          IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()),
 704                          IRB.CreateIntCast(M->getArgOperand(2), IntPtrTy, false)});
 705                 I->eraseFromParent();
 706         }
 707         return false;
 708 }
 709
 710 bool CDSPass::instrumentAtomic(Instruction * I, const DataLayout &DL) {
 711         IRBuilder<> IRB(I);
 712
 713         if (auto *CI = dyn_cast<CallInst>(I)) {
 714                 return instrumentAtomicCall(CI, DL);
 715         }
 716
 717         Value *position = getPosition(I, IRB);
 718
 719         if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
 720                 Value *Addr = LI->getPointerOperand();
 721                 int Idx=getMemoryAccessFuncIndex(Addr, DL);
 722                 if (Idx < 0)
 723                         return false;
 724
 725                 int atomic_order_index = getAtomicOrderIndex(LI->getOrdering());
 726                 Value *order = ConstantInt::get(OrdTy, atomic_order_index);
 727                 Value *args[] = {Addr, order, position};
 728                 Instruction* funcInst = CallInst::Create(CDSAtomicLoad[Idx], args);
 729                 ReplaceInstWithInst(LI, funcInst);
 730         } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
 731                 Value *Addr = SI->getPointerOperand();
 732                 int Idx=getMemoryAccessFuncIndex(Addr, DL);
 733                 if (Idx < 0)
 734                         return false;
 735
 736                 int atomic_order_index = getAtomicOrderIndex(SI->getOrdering());
 737                 Value *val = SI->getValueOperand();
 738                 Value *order = ConstantInt::get(OrdTy, atomic_order_index);
 739                 Value *args[] = {Addr, val, order, position};
 740                 Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
 741                 ReplaceInstWithInst(SI, funcInst);
 742         } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) {
 743                 Value *Addr = RMWI->getPointerOperand();
 744                 int Idx=getMemoryAccessFuncIndex(Addr, DL);
 745                 if (Idx < 0)
 746                         return false;
 747
 748                 int atomic_order_index = getAtomicOrderIndex(RMWI->getOrdering());
 749                 Value *val = RMWI->getValOperand();
 750                 Value *order = ConstantInt::get(OrdTy, atomic_order_index);
 751                 Value *args[] = {Addr, val, order, position};
 752                 Instruction* funcInst = CallInst::Create(CDSAtomicRMW[RMWI->getOperation()][Idx], args);
 753                 ReplaceInstWithInst(RMWI, funcInst);
 754         } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) {
 755                 IRBuilder<> IRB(CASI);
 756
 757                 Value *Addr = CASI->getPointerOperand();
 758                 int Idx=getMemoryAccessFuncIndex(Addr, DL);
 759                 if (Idx < 0)
 760                         return false;
 761
 762                 const unsigned ByteSize = 1U << Idx;
 763                 const unsigned BitSize = ByteSize * 8;
 764                 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize);
 765                 Type *PtrTy = Ty->getPointerTo();
 766
 767                 Value *CmpOperand = IRB.CreateBitOrPointerCast(CASI->getCompareOperand(), Ty);
 768                 Value *NewOperand = IRB.CreateBitOrPointerCast(CASI->getNewValOperand(), Ty);
 769
 770                 int atomic_order_index_succ = getAtomicOrderIndex(CASI->getSuccessOrdering());
 771                 int atomic_order_index_fail = getAtomicOrderIndex(CASI->getFailureOrdering());
 772                 Value *order_succ = ConstantInt::get(OrdTy, atomic_order_index_succ);
 773                 Value *order_fail = ConstantInt::get(OrdTy, atomic_order_index_fail);
 774
 775                 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
 776                                                  CmpOperand, NewOperand,
 777                                                  order_succ, order_fail, position};
 778
 779                 CallInst *funcInst = IRB.CreateCall(CDSAtomicCAS_V1[Idx], Args);
 780                 Value *Success = IRB.CreateICmpEQ(funcInst, CmpOperand);
 781
 782                 Value *OldVal = funcInst;
 783                 Type *OrigOldValTy = CASI->getNewValOperand()->getType();
 784                 if (Ty != OrigOldValTy) {
 785                         // The value is a pointer, so we need to cast the return value.
 786                         OldVal = IRB.CreateIntToPtr(funcInst, OrigOldValTy);
 787                 }
 788
 789                 Value *Res =
 790                   IRB.CreateInsertValue(UndefValue::get(CASI->getType()), OldVal, 0);
 791                 Res = IRB.CreateInsertValue(Res, Success, 1);
 792
 793                 I->replaceAllUsesWith(Res);
 794                 I->eraseFromParent();
 795         } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) {
 796                 int atomic_order_index = getAtomicOrderIndex(FI->getOrdering());
 797                 Value *order = ConstantInt::get(OrdTy, atomic_order_index);
 798                 Value *Args[] = {order, position};
 799
 800                 CallInst *funcInst = CallInst::Create(CDSAtomicThreadFence, Args);
 801                 ReplaceInstWithInst(FI, funcInst);
 802                 // errs() << "Thread Fences replaced\n";
 803         }
 804         return true;
 805 }
 806
 807 bool CDSPass::isAtomicCall(Instruction *I) {
 808         if ( auto *CI = dyn_cast<CallInst>(I) ) {
 809                 Function *fun = CI->getCalledFunction();
 810                 if (fun == NULL)
 811                         return false;
 812
 813                 StringRef funName = fun->getName();
 814
 815                 // TODO: come up with better rules for function name checking
 816                 for (StringRef name : AtomicFuncNames) {
 817                         if ( funName.contains(name) )
 818                                 return true;
 819                 }
 820
 821                 for (StringRef PartialName : PartialAtomicFuncNames) {
 822                         if (funName.contains(PartialName) &&
 823                                         funName.contains("atomic") )
 824                                 return true;
 825                 }
 826         }
 827
 828         return false;
 829 }
 830
 831 bool CDSPass::instrumentAtomicCall(CallInst *CI, const DataLayout &DL) {
 832         IRBuilder<> IRB(CI);
 833         Function *fun = CI->getCalledFunction();
 834         StringRef funName = fun->getName();
 835         std::vector<Value *> parameters;
 836
 837         User::op_iterator begin = CI->arg_begin();
 838         User::op_iterator end = CI->arg_end();
 839         for (User::op_iterator it = begin; it != end; ++it) {
 840                 Value *param = *it;
 841                 parameters.push_back(param);
 842         }
 843
 844         // obtain source line number of the CallInst
 845         Value *position = getPosition(CI, IRB);
 846
 847         // the pointer to the address is always the first argument
 848         Value *OrigPtr = parameters[0];
 849
 850         int Idx = getMemoryAccessFuncIndex(OrigPtr, DL);
 851         if (Idx < 0)
 852                 return false;
 853
 854         const unsigned ByteSize = 1U << Idx;
 855         const unsigned BitSize = ByteSize * 8;
 856         Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize);
 857         Type *PtrTy = Ty->getPointerTo();
 858
 859         // atomic_init; args = {obj, order}
 860         if (funName.contains("atomic_init")) {
 861                 Value *OrigVal = parameters[1];
 862
 863                 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
 864                 Value *val;
 865                 if (OrigVal->getType()->isPtrOrPtrVectorTy())
 866                         val = IRB.CreatePointerCast(OrigVal, Ty);
 867                 else
 868                         val = IRB.CreateIntCast(OrigVal, Ty, true);
 869
 870                 Value *args[] = {ptr, val, position};
 871
 872                 Instruction* funcInst = CallInst::Create(CDSAtomicInit[Idx], args);
 873                 ReplaceInstWithInst(CI, funcInst);
 874
 875                 return true;
 876         }
 877
 878         // atomic_load; args = {obj, order}
 879         if (funName.contains("atomic_load")) {
 880                 bool isExplicit = funName.contains("atomic_load_explicit");
 881
 882                 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
 883                 Value *order;
 884                 if (isExplicit)
 885                         order = IRB.CreateBitOrPointerCast(parameters[1], OrdTy);
 886                 else
 887                         order = ConstantInt::get(OrdTy,
 888                                                         (int) AtomicOrderingCABI::seq_cst);
 889                 Value *args[] = {ptr, order, position};
 890
 891                 Instruction* funcInst = CallInst::Create(CDSAtomicLoad[Idx], args);
 892                 ReplaceInstWithInst(CI, funcInst);
 893
 894                 return true;
 895         } else if (funName.contains("atomic") &&
 896                                         funName.contains("load") ) {
 897                 // does this version of call always have an atomic order as an argument?
 898                 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
 899                 Value *order = IRB.CreateBitOrPointerCast(parameters[1], OrdTy);
 900                 Value *args[] = {ptr, order, position};
 901
 902                 if (!CI->getType()->isPointerTy()) {
 903                         return false;
 904                 }
 905
 906                 CallInst *funcInst = IRB.CreateCall(CDSAtomicLoad[Idx], args);
 907                 Value *RetVal = IRB.CreateIntToPtr(funcInst, CI->getType());
 908
 909                 CI->replaceAllUsesWith(RetVal);
 910                 CI->eraseFromParent();
 911
 912                 return true;
 913         }
 914
 915         // atomic_store; args = {obj, val, order}
 916         if (funName.contains("atomic_store")) {
 917                 bool isExplicit = funName.contains("atomic_store_explicit");
 918                 Value *OrigVal = parameters[1];
 919
 920                 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
 921                 Value *val = IRB.CreatePointerCast(OrigVal, Ty);
 922                 Value *order;
 923                 if (isExplicit)
 924                         order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
 925                 else
 926                         order = ConstantInt::get(OrdTy,
 927                                                         (int) AtomicOrderingCABI::seq_cst);
 928                 Value *args[] = {ptr, val, order, position};
 929
 930                 Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
 931                 ReplaceInstWithInst(CI, funcInst);
 932
 933                 return true;
 934         } else if (funName.contains("atomic") &&
 935                                         funName.contains("store") ) {
 936                 // Does this version of call always have an atomic order as an argument?
 937                 Value *OrigVal = parameters[1];
 938
 939                 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
 940                 Value *val;
 941                 if (OrigVal->getType()->isPtrOrPtrVectorTy())
 942                         val = IRB.CreatePointerCast(OrigVal, Ty);
 943                 else
 944                         val = IRB.CreateIntCast(OrigVal, Ty, true);
 945
 946                 Value *order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
 947                 Value *args[] = {ptr, val, order, position};
 948
 949                 Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
 950                 ReplaceInstWithInst(CI, funcInst);
 951
 952                 return true;
 953         }
 954
 955         // atomic_fetch_*; args = {obj, val, order}
 956         if (funName.contains("atomic_fetch_") ||
 957                 funName.contains("atomic_exchange")) {
 958
 959                 /* TODO: implement stricter function name checking */
 960                 if (funName.contains("non"))
 961                         return false;
 962
 963                 bool isExplicit = funName.contains("_explicit");
 964                 Value *OrigVal = parameters[1];
 965
 966                 int op;
 967                 if ( funName.contains("_fetch_add") )
 968                         op = AtomicRMWInst::Add;
 969                 else if ( funName.contains("_fetch_sub") )
 970                         op = AtomicRMWInst::Sub;
 971                 else if ( funName.contains("_fetch_and") )
 972                         op = AtomicRMWInst::And;
 973                 else if ( funName.contains("_fetch_or") )
 974                         op = AtomicRMWInst::Or;
 975                 else if ( funName.contains("_fetch_xor") )
 976                         op = AtomicRMWInst::Xor;
 977                 else if ( funName.contains("atomic_exchange") )
 978                         op = AtomicRMWInst::Xchg;
 979                 else {
 980                         errs() << "Unknown atomic read-modify-write operation\n";
 981                         return false;
 982                 }
 983
 984                 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
 985                 Value *val;
 986                 if (OrigVal->getType()->isPtrOrPtrVectorTy())
 987                         val = IRB.CreatePointerCast(OrigVal, Ty);
 988                 else
 989                         val = IRB.CreateIntCast(OrigVal, Ty, true);
 990
 991                 Value *order;
 992                 if (isExplicit)
 993                         order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
 994                 else
 995                         order = ConstantInt::get(OrdTy,
 996                                                         (int) AtomicOrderingCABI::seq_cst);
 997                 Value *args[] = {ptr, val, order, position};
 998
 999                 Instruction* funcInst = CallInst::Create(CDSAtomicRMW[op][Idx], args);
1000                 ReplaceInstWithInst(CI, funcInst);
1001
1002                 return true;
1003         } else if (funName.contains("fetch")) {
1004                 errs() << "atomic fetch captured. Not implemented yet. ";
1005                 errs() << "See source file :";
1006                 getPosition(CI, IRB, true);
1007                 return false;
1008         } else if (funName.contains("exchange") &&
1009                         !funName.contains("compare_exchange") ) {
1010                 if (CI->getType()->isPointerTy()) {
1011                         /**
1012                          * TODO: instrument the following case
1013                          * mcs-lock.h
1014                          * std::atomic<struct T *> m_tail;
1015                          * struct T * me;
1016                          * struct T * pred = m_tail.exchange(me, memory_order_*);
1017                          */
1018                         errs() << "atomic exchange captured. Not implemented yet. ";
1019                         errs() << "See source file :";
1020                         getPosition(CI, IRB, true);
1021
1022                         return false;
1023                 }
1024
1025                 Value *OrigVal = parameters[1];
1026
1027                 Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
1028                 Value *val;
1029                 if (OrigVal->getType()->isPtrOrPtrVectorTy())
1030                         val = IRB.CreatePointerCast(OrigVal, Ty);
1031                 else
1032                         val = IRB.CreateIntCast(OrigVal, Ty, true);
1033
1034                 Value *order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
1035                 Value *args[] = {ptr, val, order, position};
1036                 int op = AtomicRMWInst::Xchg;
1037
1038                 Instruction* funcInst = CallInst::Create(CDSAtomicRMW[op][Idx], args);
1039                 ReplaceInstWithInst(CI, funcInst);
1040
1041                 return true;
1042         }
1043
1044         /* atomic_compare_exchange_*;
1045            args = {obj, expected, new value, order1, order2}
1046         */
1047         if ( funName.contains("atomic_compare_exchange_") ) {
1048                 bool isExplicit = funName.contains("_explicit");
1049
1050                 Value *Addr = IRB.CreatePointerCast(OrigPtr, PtrTy);
1051                 Value *CmpOperand = IRB.CreatePointerCast(parameters[1], PtrTy);
1052                 Value *NewOperand = IRB.CreateBitOrPointerCast(parameters[2], Ty);
1053
1054                 Value *order_succ, *order_fail;
1055                 if (isExplicit) {
1056                         order_succ = IRB.CreateBitOrPointerCast(parameters[3], OrdTy);
1057
1058                         if (parameters.size() > 4) {
1059                                 order_fail = IRB.CreateBitOrPointerCast(parameters[4], OrdTy);
1060                         } else {
1061                                 /* The failure order is not provided */
1062                                 order_fail = order_succ;
1063                                 ConstantInt * order_succ_cast = dyn_cast<ConstantInt>(order_succ);
1064                                 int index = order_succ_cast->getSExtValue();
1065
1066                                 order_fail = ConstantInt::get(OrdTy,
1067                                                                 AtomicCasFailureOrderIndex(index));
1068                         }
1069                 } else  {
1070                         order_succ = ConstantInt::get(OrdTy,
1071                                                         (int) AtomicOrderingCABI::seq_cst);
1072                         order_fail = ConstantInt::get(OrdTy,
1073                                                         (int) AtomicOrderingCABI::seq_cst);
1074                 }
1075
1076                 Value *args[] = {Addr, CmpOperand, NewOperand,
1077                                                         order_succ, order_fail, position};
1078
1079                 Instruction* funcInst = CallInst::Create(CDSAtomicCAS_V2[Idx], args);
1080                 ReplaceInstWithInst(CI, funcInst);
1081
1082                 return true;
1083         } else if ( funName.contains("compare_exchange_strong") ||
1084                                 funName.contains("compare_exchange_weak") ) {
1085                 Value *Addr = IRB.CreatePointerCast(OrigPtr, PtrTy);
1086                 Value *CmpOperand = IRB.CreatePointerCast(parameters[1], PtrTy);
1087                 Value *NewOperand = IRB.CreateBitOrPointerCast(parameters[2], Ty);
1088
1089                 Value *order_succ, *order_fail;
1090                 order_succ = IRB.CreateBitOrPointerCast(parameters[3], OrdTy);
1091
1092                 if (parameters.size() > 4) {
1093                         order_fail = IRB.CreateBitOrPointerCast(parameters[4], OrdTy);
1094                 } else {
1095                         /* The failure order is not provided */
1096                         order_fail = order_succ;
1097                         ConstantInt * order_succ_cast = dyn_cast<ConstantInt>(order_succ);
1098                         int index = order_succ_cast->getSExtValue();
1099
1100                         order_fail = ConstantInt::get(OrdTy,
1101                                                         AtomicCasFailureOrderIndex(index));
1102                 }
1103
1104                 Value *args[] = {Addr, CmpOperand, NewOperand,
1105                                                         order_succ, order_fail, position};
1106                 Instruction* funcInst = CallInst::Create(CDSAtomicCAS_V2[Idx], args);
1107                 ReplaceInstWithInst(CI, funcInst);
1108
1109                 return true;
1110         }
1111
1112         return false;
1113 }
1114
1115 int CDSPass::getMemoryAccessFuncIndex(Value *Addr,
1116                                                                                 const DataLayout &DL) {
1117         Type *OrigPtrTy = Addr->getType();
1118         Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType();
1119         assert(OrigTy->isSized());
1120         uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy);
1121         if (TypeSize != 8  && TypeSize != 16 &&
1122                 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) {
1123                 NumAccessesWithBadSize++;
1124                 // Ignore all unusual sizes.
1125                 return -1;
1126         }
1127         size_t Idx = countTrailingZeros(TypeSize / 8);
1128         //assert(Idx < kNumberOfAccessSizes);
1129         if (Idx >= kNumberOfAccessSizes) {
1130                 return -1;
1131         }
1132         return Idx;
1133 }
1134
1135
1136 char CDSPass::ID = 0;
1137
1138 // Automatically enable the pass.
1139 static void registerCDSPass(const PassManagerBuilder &,
1140                                                         legacy::PassManagerBase &PM) {
1141         PM.add(new CDSPass());
1142 }
1143
1144 /* Enable the pass when opt level is greater than 0 */
1145 static RegisterStandardPasses
1146         RegisterMyPass1(PassManagerBuilder::EP_OptimizerLast,
1147 registerCDSPass);
1148
1149 /* Enable the pass when opt level is 0 */
1150 static RegisterStandardPasses
1151         RegisterMyPass2(PassManagerBuilder::EP_EnabledOnOptLevel0,
1152 registerCDSPass);