From d31096383384094e6162803bb98408a3be444f7b Mon Sep 17 00:00:00 2001 From: Robin Morisset Date: Thu, 16 Oct 2014 20:34:57 +0000 Subject: [PATCH] Erase fence insertion from SelectionDAGBuilder.cpp (NFC) Summary: Backends can use setInsertFencesForAtomic to signal to the middle-end that montonic is the only memory ordering they can accept for stores/loads/rmws/cmpxchg. The code lowering those accesses with a stronger ordering to fences + monotonic accesses is currently living in SelectionDAGBuilder.cpp. In this patch I propose moving this logic out of it for several reasons: - There is lots of redundancy to avoid: extremely similar logic already exists in AtomicExpand. - The current code in SelectionDAGBuilder does not use any target-hooks, it does the same transformation for every backend that requires it - As a result it is plain *unsound*, as it was apparently designed for ARM. It happens to mostly work for the other targets because they are extremely conservative, but Power for example had to switch to AtomicExpand to be able to use lwsync safely (see r218331). - Because it produces IR-level fences, it cannot be made sound ! This is noted in the C++11 standard (section 29.3, page 1140): ``` Fences cannot, in general, be used to restore sequential consistency for atomic operations with weaker ordering semantics. ``` It can also be seen by the following example (called IRIW in the litterature): ``` atomic x = y = 0; int r1, r2, r3, r4; Thread 0: x.store(1); Thread 1: y.store(1); Thread 2: r1 = x.load(); r2 = y.load(); Thread 3: r3 = y.load(); r4 = x.load(); ``` r1 = r3 = 1 and r2 = r4 = 0 is impossible as long as the accesses are all seq_cst. But if they are lowered to monotonic accesses, no amount of fences can prevent it.. This patch does three things (I could cut it into parts, but then some of them would not be tested/testable, please tell me if you would prefer that): - it provides a default implementation for emitLeadingFence/emitTrailingFence in terms of IR-level fences, that mimic the original logic of SelectionDAGBuilder. As we saw above, this is unsound, but the best that can be done without knowing the targets well (and there is a comment warning about this risk). - it then switches Mips/Sparc/XCore to use AtomicExpand, relying on this default implementation (that exactly replicates the logic of SelectionDAGBuilder, so no functional change) - it finally erase this logic from SelectionDAGBuilder as it is dead-code. Ideally, each target would define its own override for emitLeading/TrailingFence using target-specific fences, but I do not know the Sparc/Mips/XCore memory model well enough to do this, and they appear to be dealing fine with the ARM-inspired default expansion for now (probably because they are overly conservative, as Power was). If anyone wants to compile fences more agressively on these platforms, the long comment should make it clear why he should first override emitLeading/TrailingFence. Test Plan: make check-all, no functional change Reviewers: jfb, t.p.northover Subscribers: aemerson, llvm-commits Differential Revision: http://reviews.llvm.org/D5474 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219957 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/Target/TargetLowering.h | 51 ++++++++--- .../SelectionDAG/SelectionDAGBuilder.cpp | 87 +++++-------------- lib/Target/Mips/MipsTargetMachine.cpp | 1 + lib/Target/Sparc/SparcTargetMachine.cpp | 7 ++ lib/Target/XCore/XCoreTargetMachine.cpp | 7 ++ test/CodeGen/XCore/atomic.ll | 5 +- 6 files changed, 75 insertions(+), 83 deletions(-) diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h index 774e7d8ac8f..c6eac43de95 100644 --- a/include/llvm/Target/TargetLowering.h +++ b/include/llvm/Target/TargetLowering.h @@ -964,29 +964,54 @@ public: /// It is called by AtomicExpandPass before expanding an /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad. /// RMW and CmpXchg set both IsStore and IsLoad to true. - /// Backends with !getInsertFencesForAtomic() should keep a no-op here. /// This function should either return a nullptr, or a pointer to an IR-level /// Instruction*. Even complex fence sequences can be represented by a /// single Instruction* through an intrinsic to be lowered later. + /// Backends with !getInsertFencesForAtomic() should keep a no-op here. + /// Backends should override this method to produce target-specific intrinsic + /// for their fences. + /// FIXME: Please note that the default implementation here in terms of + /// IR-level fences exists for historical/compatibility reasons and is + /// *unsound* ! Fences cannot, in general, be used to restore sequential + /// consistency. For example, consider the following example: + /// atomic x = y = 0; + /// int r1, r2, r3, r4; + /// Thread 0: + /// x.store(1); + /// Thread 1: + /// y.store(1); + /// Thread 2: + /// r1 = x.load(); + /// r2 = y.load(); + /// Thread 3: + /// r3 = y.load(); + /// r4 = x.load(); + /// r1 = r3 = 1 and r2 = r4 = 0 is impossible as long as the accesses are all + /// seq_cst. But if they are lowered to monotonic accesses, no amount of + /// IR-level fences can prevent it. + /// @{ virtual Instruction* emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const { - assert(!getInsertFencesForAtomic()); - return nullptr; + if (!getInsertFencesForAtomic()) + return nullptr; + + if (isAtLeastRelease(Ord) && IsStore) + return Builder.CreateFence(Ord); + else + return nullptr; } - /// Inserts in the IR a target-specific intrinsic specifying a fence. - /// It is called by AtomicExpandPass after expanding an - /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad. - /// RMW and CmpXchg set both IsStore and IsLoad to true. - /// Backends with !getInsertFencesForAtomic() should keep a no-op here. - /// This function should either return a nullptr, or a pointer to an IR-level - /// Instruction*. Even complex fence sequences can be represented by a - /// single Instruction* through an intrinsic to be lowered later. virtual Instruction* emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const { - assert(!getInsertFencesForAtomic()); - return nullptr; + if (!getInsertFencesForAtomic()) + return nullptr; + + if (isAtLeastAcquire(Ord)) + return Builder.CreateFence(Ord); + else + return nullptr; } + /// @} /// Returns true if the given (atomic) store should be expanded by the /// IR-level AtomicExpand pass into an "atomic xchg" which ignores its input. diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index a51ed2da277..32b0dde8138 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3604,30 +3604,6 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { DAG.setRoot(StoreNode); } -static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order, - SynchronizationScope Scope, - bool Before, SDLoc dl, - SelectionDAG &DAG, - const TargetLowering &TLI) { - // Fence, if necessary - if (Before) { - if (Order == AcquireRelease || Order == SequentiallyConsistent) - Order = Release; - else if (Order == Acquire || Order == Monotonic || Order == Unordered) - return Chain; - } else { - if (Order == AcquireRelease) - Order = Acquire; - else if (Order == Release || Order == Monotonic || Order == Unordered) - return Chain; - } - SDValue Ops[3]; - Ops[0] = Chain; - Ops[1] = DAG.getConstant(Order, TLI.getPointerTy()); - Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy()); - return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops); -} - void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { SDLoc dl = getCurSDLoc(); AtomicOrdering SuccessOrder = I.getSuccessOrdering(); @@ -3636,27 +3612,16 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { SDValue InChain = getRoot(); - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - if (TLI.getInsertFencesForAtomic()) - InChain = - InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl, DAG, TLI); - MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType(); SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); SDValue L = DAG.getAtomicCmpSwap( ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain, getValue(I.getPointerOperand()), getValue(I.getCompareOperand()), getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()), - 0 /* Alignment */, - TLI.getInsertFencesForAtomic() ? Monotonic : SuccessOrder, - TLI.getInsertFencesForAtomic() ? Monotonic : FailureOrder, Scope); + /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope); SDValue OutChain = L.getValue(2); - if (TLI.getInsertFencesForAtomic()) - OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl, - DAG, TLI); - setValue(&I, L); DAG.setRoot(OutChain); } @@ -3683,22 +3648,17 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { SDValue InChain = getRoot(); - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - if (TLI.getInsertFencesForAtomic()) - InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, DAG, TLI); - - SDValue L = DAG.getAtomic( - NT, dl, getValue(I.getValOperand()).getSimpleValueType(), InChain, - getValue(I.getPointerOperand()), getValue(I.getValOperand()), - I.getPointerOperand(), 0 /* Alignment */, - TLI.getInsertFencesForAtomic() ? Monotonic : Order, Scope); + SDValue L = + DAG.getAtomic(NT, dl, + getValue(I.getValOperand()).getSimpleValueType(), + InChain, + getValue(I.getPointerOperand()), + getValue(I.getValOperand()), + I.getPointerOperand(), + /* Alignment=*/ 0, Order, Scope); SDValue OutChain = L.getValue(1); - if (TLI.getInsertFencesForAtomic()) - OutChain = - InsertFenceForAtomic(OutChain, Order, Scope, false, dl, DAG, TLI); - setValue(&I, L); DAG.setRoot(OutChain); } @@ -3736,16 +3696,13 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { DAG.getEVTAlignment(VT)); InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); - SDValue L = DAG.getAtomic( - ISD::ATOMIC_LOAD, dl, VT, VT, InChain, getValue(I.getPointerOperand()), - MMO, TLI.getInsertFencesForAtomic() ? Monotonic : Order, Scope); + SDValue L = + DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, + getValue(I.getPointerOperand()), MMO, + Order, Scope); SDValue OutChain = L.getValue(1); - if (TLI.getInsertFencesForAtomic()) - OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, - DAG, TLI); - setValue(&I, L); DAG.setRoot(OutChain); } @@ -3764,17 +3721,13 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { if (I.getAlignment() < VT.getSizeInBits() / 8) report_fatal_error("Cannot generate unaligned atomic store"); - if (TLI.getInsertFencesForAtomic()) - InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, DAG, TLI); - - SDValue OutChain = DAG.getAtomic( - ISD::ATOMIC_STORE, dl, VT, InChain, getValue(I.getPointerOperand()), - getValue(I.getValueOperand()), I.getPointerOperand(), I.getAlignment(), - TLI.getInsertFencesForAtomic() ? Monotonic : Order, Scope); - - if (TLI.getInsertFencesForAtomic()) - OutChain = - InsertFenceForAtomic(OutChain, Order, Scope, false, dl, DAG, TLI); + SDValue OutChain = + DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, + InChain, + getValue(I.getPointerOperand()), + getValue(I.getValueOperand()), + I.getPointerOperand(), I.getAlignment(), + Order, Scope); DAG.setRoot(OutChain); } diff --git a/lib/Target/Mips/MipsTargetMachine.cpp b/lib/Target/Mips/MipsTargetMachine.cpp index d453935d3e1..dd6c8cbef03 100644 --- a/lib/Target/Mips/MipsTargetMachine.cpp +++ b/lib/Target/Mips/MipsTargetMachine.cpp @@ -178,6 +178,7 @@ TargetPassConfig *MipsTargetMachine::createPassConfig(PassManagerBase &PM) { void MipsPassConfig::addIRPasses() { TargetPassConfig::addIRPasses(); + addPass(createAtomicExpandPass(&getMipsTargetMachine())); if (getMipsSubtarget().os16()) addPass(createMipsOs16(getMipsTargetMachine())); if (getMipsSubtarget().inMips16HardFloat()) diff --git a/lib/Target/Sparc/SparcTargetMachine.cpp b/lib/Target/Sparc/SparcTargetMachine.cpp index 80c71448963..ae481b92dea 100644 --- a/lib/Target/Sparc/SparcTargetMachine.cpp +++ b/lib/Target/Sparc/SparcTargetMachine.cpp @@ -47,6 +47,7 @@ public: return getTM(); } + void addIRPasses() override; bool addInstSelector() override; bool addPreEmitPass() override; }; @@ -56,6 +57,12 @@ TargetPassConfig *SparcTargetMachine::createPassConfig(PassManagerBase &PM) { return new SparcPassConfig(this, PM); } +void SparcPassConfig::addIRPasses() { + addPass(createAtomicExpandPass(&getSparcTargetMachine())); + + TargetPassConfig::addIRPasses(); +} + bool SparcPassConfig::addInstSelector() { addPass(createSparcISelDag(getSparcTargetMachine())); return false; diff --git a/lib/Target/XCore/XCoreTargetMachine.cpp b/lib/Target/XCore/XCoreTargetMachine.cpp index 8d8bb3800ea..81925dcce06 100644 --- a/lib/Target/XCore/XCoreTargetMachine.cpp +++ b/lib/Target/XCore/XCoreTargetMachine.cpp @@ -41,6 +41,7 @@ public: return getTM(); } + void addIRPasses() override; bool addPreISel() override; bool addInstSelector() override; bool addPreEmitPass() override; @@ -51,6 +52,12 @@ TargetPassConfig *XCoreTargetMachine::createPassConfig(PassManagerBase &PM) { return new XCorePassConfig(this, PM); } +void XCorePassConfig::addIRPasses() { + addPass(createAtomicExpandPass(&getXCoreTargetMachine())); + + TargetPassConfig::addIRPasses(); +} + bool XCorePassConfig::addPreISel() { addPass(createXCoreLowerThreadLocalPass()); return false; diff --git a/test/CodeGen/XCore/atomic.ll b/test/CodeGen/XCore/atomic.ll index 58ef38bd3f6..6ca80cf5d9e 100644 --- a/test/CodeGen/XCore/atomic.ll +++ b/test/CodeGen/XCore/atomic.ll @@ -22,11 +22,10 @@ entry: ; CHECK-LABEL: atomicloadstore ; CHECK: ldw r[[R0:[0-9]+]], dp[pool] -; CHECK-NEXT: #MEMBARRIER - %0 = load atomic i32* bitcast (i64* @pool to i32*) acquire, align 4 - ; CHECK-NEXT: ldaw r[[R1:[0-9]+]], dp[pool] +; CHECK-NEXT: #MEMBARRIER ; CHECK-NEXT: ldc r[[R2:[0-9]+]], 0 + %0 = load atomic i32* bitcast (i64* @pool to i32*) acquire, align 4 ; CHECK-NEXT: ld16s r3, r[[R1]][r[[R2]]] ; CHECK-NEXT: #MEMBARRIER -- 2.34.1