Taints the non-acquire RMW's store address with the load part

[oota-llvm.git] / lib / CodeGen / AtomicExpandPass.cpp

diff --git a/lib/CodeGen/AtomicExpandPass.cpp b/lib/CodeGen/AtomicExpandPass.cpp
index b2134a7859afc1a0b3db484afafcd31661ddd57e..045c8076d7e7c4c7288a8906855435485c8392c4 100644 (file)
--- a/lib/CodeGen/AtomicExpandPass.cpp
+++ b/lib/CodeGen/AtomicExpandPass.cpp
@@ -15,6 +15,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TaintRelaxedAtomicsUtils.h"
+#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/CodeGen/AtomicExpandUtils.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/IR/Function.h"
@@ -23,11 +29,13 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/NoFolder.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 
 using namespace llvm;
 
@@ -64,6 +72,33 @@ namespace {
     bool isIdempotentRMW(AtomicRMWInst *AI);
     bool simplifyIdempotentRMW(AtomicRMWInst *AI);
   };
+
+
+  // If 'LI' is a relaxed load, and it is immediately followed by a
+// atomic read-modify-write that has acq_rel parameter, we don't have to do
+// anything since the rmw serves as a natural barrier.
+void MarkRelaxedLoadBeforeAcqrelRMW(LoadInst* LI) {
+  auto* BB = LI->getParent();
+  auto BBI = LI->getIterator();
+  for (BBI++; BBI != BB->end(); BBI++) {
+    Instruction* CurInst = &*BBI;
+    if (!CurInst) {
+      return;
+    }
+    if (!CurInst->isAtomic()) {
+      continue;
+    }
+    auto* RMW = dyn_cast<AtomicRMWInst>(CurInst);
+    if (!RMW) {
+      return;
+    }
+    if (RMW->getOrdering() == AcquireRelease ||
+        RMW->getOrdering() == SequentiallyConsistent) {
+      LI->setHasSubsequentAcqlRMW(true);
+    }
+  }
+}
+
 }
 
 char AtomicExpand::ID = 0;
@@ -82,6 +117,7 @@ bool AtomicExpand::runOnFunction(Function &F) {
   TLI = TM->getSubtargetImpl(F)->getTargetLowering();
 
   SmallVector<Instruction *, 1> AtomicInsts;
+  SmallVector<LoadInst*, 1> MonotonicLoadInsts;
 
   // Changing control-flow while iterating through it is a bad idea, so gather a
   // list of all atomic instructions before we start.
@@ -91,6 +127,10 @@ bool AtomicExpand::runOnFunction(Function &F) {
     if (I->isAtomic()) {
       switch (I->getOpcode()) {
         case Instruction::AtomicCmpXchg: {
+          // XXX-comment: AtomicCmpXchg in AArch64 will be translated to a
+          // conditional branch that contains the value of the load anyway, so
+          // we don't need to do anything.
+          /*
           auto* CmpXchg = dyn_cast<AtomicCmpXchgInst>(&*I);
           auto SuccOrdering = CmpXchg->getSuccessOrdering();
           if (SuccOrdering == Monotonic) {
@@ -98,19 +138,31 @@ bool AtomicExpand::runOnFunction(Function &F) {
           } else if (SuccOrdering == Release) {
             CmpXchg->setSuccessOrdering(AcquireRelease);
           }
+          */
           break;
         }
         case Instruction::AtomicRMW: {
+          // XXX-comment: Similar to AtomicCmpXchg. These instructions in
+          // AArch64 will be translated to a loop whose condition depends on the
+          // store status, which further depends on the load value.
+          /*
           auto* RMW = dyn_cast<AtomicRMWInst>(&*I);
           if (RMW->getOrdering() == Monotonic) {
             RMW->setOrdering(Acquire);
           }
+          */
           break;
         }
         case Instruction::Load: {
           auto* LI = dyn_cast<LoadInst>(&*I);
           if (LI->getOrdering() == Monotonic) {
+            /*
+            DEBUG(dbgs() << "Transforming relaxed loads to acquire loads: "
+                         << *LI << '\n');
             LI->setOrdering(Acquire);
+            */
+//            MonotonicLoadInsts.push_back(LI);
+            MarkRelaxedLoadBeforeAcqrelRMW(LI);
           }
           break;
         }
@@ -136,7 +188,7 @@ bool AtomicExpand::runOnFunction(Function &F) {
     if (TLI->getInsertFencesForAtomic()) {
       if (LI && isAtLeastAcquire(LI->getOrdering())) {
         FenceOrdering = LI->getOrdering();
-        LI->setOrdering(Monotonic);
+//        AddFakeConditionalBranch(
         IsStore = false;
         IsLoad = true;
       } else if (SI && isAtLeastRelease(SI->getOrdering())) {
@@ -204,6 +256,7 @@ bool AtomicExpand::runOnFunction(Function &F) {
       MadeChange |= expandAtomicCmpXchg(CASI);
     }
   }
+
   return MadeChange;
 }
 
@@ -464,10 +517,25 @@ bool AtomicExpand::expandAtomicOpToLLSC(
   Builder.SetInsertPoint(LoopBB);
   Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
 
+  // XXX-update: For relaxed RMWs (i.e., fetch_* operations), we still need to
+  // taint the load part. However, we only need to taint those whose results are
+  // not immediately used by a conditional branch or a store address.
+  Value* StoreAddr = Addr;
+  auto* LoadedPartInst = dyn_cast<Instruction>(Loaded);
+  assert(LoadedPartInst && "Load part of RMW should be an instruction!");
+  if (MemOpOrder != Acquire && MemOpOrder != AcquireRelease &&
+      MemOpOrder != SequentiallyConsistent) {
+    // Also check whether the result is used immediately. If so, taint the
+    // address of the upcoming store-exclusive.
+    if (NeedExtraConstraints(I)) {
+      StoreAddr = taintRMWStoreAddressWithLoadPart(Builder, Addr, LoadedPartInst);
+    }
+  }
+
   Value *NewVal = PerformOp(Builder, Loaded);
 
   Value *StoreSuccess =
-      TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
+      TLI->emitStoreConditional(Builder, NewVal, StoreAddr, MemOpOrder);
   Value *TryAgain = Builder.CreateICmpNE(
       StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain");
   Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);