-static void getExclusiveOperation(unsigned Size, AtomicOrdering Ord,
- unsigned &LdrOpc, unsigned &StrOpc) {
- static unsigned LoadBares[] = { ARM64::LDXRB, ARM64::LDXRH, ARM64::LDXRW,
- ARM64::LDXRX, ARM64::LDXPX };
- static unsigned LoadAcqs[] = { ARM64::LDAXRB, ARM64::LDAXRH, ARM64::LDAXRW,
- ARM64::LDAXRX, ARM64::LDAXPX };
- static unsigned StoreBares[] = { ARM64::STXRB, ARM64::STXRH, ARM64::STXRW,
- ARM64::STXRX, ARM64::STXPX };
- static unsigned StoreRels[] = { ARM64::STLXRB, ARM64::STLXRH, ARM64::STLXRW,
- ARM64::STLXRX, ARM64::STLXPX };
-
- unsigned *LoadOps, *StoreOps;
- if (Ord == Acquire || Ord == AcquireRelease || Ord == SequentiallyConsistent)
- LoadOps = LoadAcqs;
- else
- LoadOps = LoadBares;
-
- if (Ord == Release || Ord == AcquireRelease || Ord == SequentiallyConsistent)
- StoreOps = StoreRels;
- else
- StoreOps = StoreBares;
-
- assert(isPowerOf2_32(Size) && Size <= 16 &&
- "unsupported size for atomic binary op!");
-
- LdrOpc = LoadOps[Log2_32(Size)];
- StrOpc = StoreOps[Log2_32(Size)];
-}
-
-MachineBasicBlock *ARM64TargetLowering::EmitAtomicCmpSwap(MachineInstr *MI,
- MachineBasicBlock *BB,
- unsigned Size) const {
- unsigned dest = MI->getOperand(0).getReg();
- unsigned ptr = MI->getOperand(1).getReg();
- unsigned oldval = MI->getOperand(2).getReg();
- unsigned newval = MI->getOperand(3).getReg();
- AtomicOrdering Ord = static_cast<AtomicOrdering>(MI->getOperand(4).getImm());
- unsigned scratch = BB->getParent()->getRegInfo().createVirtualRegister(
- &ARM64::GPR32RegClass);
- const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
- DebugLoc dl = MI->getDebugLoc();
-
- // FIXME: We currently always generate a seq_cst operation; we should
- // be able to relax this in some cases.
- unsigned ldrOpc, strOpc;
- getExclusiveOperation(Size, Ord, ldrOpc, strOpc);
-
- MachineFunction *MF = BB->getParent();
- const BasicBlock *LLVM_BB = BB->getBasicBlock();
- MachineFunction::iterator It = BB;
- ++It; // insert the new blocks after the current block
-
- MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MF->insert(It, loop1MBB);
- MF->insert(It, loop2MBB);
- MF->insert(It, exitMBB);
-
- // Transfer the remainder of BB and its successor edges to exitMBB.
- exitMBB->splice(exitMBB->begin(), BB,
- std::next(MachineBasicBlock::iterator(MI)), BB->end());
- exitMBB->transferSuccessorsAndUpdatePHIs(BB);
-
- // thisMBB:
- // ...
- // fallthrough --> loop1MBB
- BB->addSuccessor(loop1MBB);
-
- // loop1MBB:
- // ldrex dest, [ptr]
- // cmp dest, oldval
- // bne exitMBB
- BB = loop1MBB;
- BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr);
- BuildMI(BB, dl, TII->get(Size == 8 ? ARM64::SUBSXrr : ARM64::SUBSWrr))
- .addReg(Size == 8 ? ARM64::XZR : ARM64::WZR, RegState::Define)
- .addReg(dest)
- .addReg(oldval);
- BuildMI(BB, dl, TII->get(ARM64::Bcc)).addImm(ARM64CC::NE).addMBB(exitMBB);
- BB->addSuccessor(loop2MBB);
- BB->addSuccessor(exitMBB);
-
- // loop2MBB:
- // strex scratch, newval, [ptr]
- // cmp scratch, #0
- // bne loop1MBB
- BB = loop2MBB;
- BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(newval).addReg(ptr);
- BuildMI(BB, dl, TII->get(ARM64::CBNZW)).addReg(scratch).addMBB(loop1MBB);
- BB->addSuccessor(loop1MBB);
- BB->addSuccessor(exitMBB);
-
- // exitMBB:
- // ...
- BB = exitMBB;
-
- MI->eraseFromParent(); // The instruction is gone now.
-
- return BB;
-}
-
-MachineBasicBlock *
-ARM64TargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
- unsigned Size, unsigned BinOpcode) const {
- // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
- const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
-
- const BasicBlock *LLVM_BB = BB->getBasicBlock();
- MachineFunction *MF = BB->getParent();
- MachineFunction::iterator It = BB;
- ++It;
-
- unsigned dest = MI->getOperand(0).getReg();
- unsigned ptr = MI->getOperand(1).getReg();
- unsigned incr = MI->getOperand(2).getReg();
- AtomicOrdering Ord = static_cast<AtomicOrdering>(MI->getOperand(3).getImm());
- DebugLoc dl = MI->getDebugLoc();
-
- unsigned ldrOpc, strOpc;
- getExclusiveOperation(Size, Ord, ldrOpc, strOpc);
-
- MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MF->insert(It, loopMBB);
- MF->insert(It, exitMBB);
-
- // Transfer the remainder of BB and its successor edges to exitMBB.
- exitMBB->splice(exitMBB->begin(), BB,
- std::next(MachineBasicBlock::iterator(MI)), BB->end());
- exitMBB->transferSuccessorsAndUpdatePHIs(BB);
-
- MachineRegisterInfo &RegInfo = MF->getRegInfo();
- unsigned scratch = RegInfo.createVirtualRegister(&ARM64::GPR32RegClass);
- unsigned scratch2 =
- (!BinOpcode)
- ? incr
- : RegInfo.createVirtualRegister(Size == 8 ? &ARM64::GPR64RegClass
- : &ARM64::GPR32RegClass);
-
- // thisMBB:
- // ...
- // fallthrough --> loopMBB
- BB->addSuccessor(loopMBB);
-
- // loopMBB:
- // ldxr dest, ptr
- // <binop> scratch2, dest, incr
- // stxr scratch, scratch2, ptr
- // cbnz scratch, loopMBB
- // fallthrough --> exitMBB
- BB = loopMBB;
- BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr);
- if (BinOpcode) {
- // operand order needs to go the other way for NAND
- if (BinOpcode == ARM64::BICWrr || BinOpcode == ARM64::BICXrr)
- BuildMI(BB, dl, TII->get(BinOpcode), scratch2).addReg(incr).addReg(dest);
- else
- BuildMI(BB, dl, TII->get(BinOpcode), scratch2).addReg(dest).addReg(incr);
- }
-
- BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(scratch2).addReg(ptr);
- BuildMI(BB, dl, TII->get(ARM64::CBNZW)).addReg(scratch).addMBB(loopMBB);
-
- BB->addSuccessor(loopMBB);
- BB->addSuccessor(exitMBB);
-
- // exitMBB:
- // ...
- BB = exitMBB;
-
- MI->eraseFromParent(); // The instruction is gone now.
-
- return BB;
-}
-
-MachineBasicBlock *ARM64TargetLowering::EmitAtomicBinary128(
- MachineInstr *MI, MachineBasicBlock *BB, unsigned BinOpcodeLo,
- unsigned BinOpcodeHi) const {
- // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
- const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
-
- const BasicBlock *LLVM_BB = BB->getBasicBlock();
- MachineFunction *MF = BB->getParent();
- MachineFunction::iterator It = BB;
- ++It;
-
- unsigned DestLo = MI->getOperand(0).getReg();
- unsigned DestHi = MI->getOperand(1).getReg();
- unsigned Ptr = MI->getOperand(2).getReg();
- unsigned IncrLo = MI->getOperand(3).getReg();
- unsigned IncrHi = MI->getOperand(4).getReg();
- AtomicOrdering Ord = static_cast<AtomicOrdering>(MI->getOperand(5).getImm());
- DebugLoc DL = MI->getDebugLoc();
-
- unsigned LdrOpc, StrOpc;
- getExclusiveOperation(16, Ord, LdrOpc, StrOpc);
-
- MachineBasicBlock *LoopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MachineBasicBlock *ExitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MF->insert(It, LoopMBB);
- MF->insert(It, ExitMBB);
-
- // Transfer the remainder of BB and its successor edges to exitMBB.
- ExitMBB->splice(ExitMBB->begin(), BB,
- std::next(MachineBasicBlock::iterator(MI)), BB->end());
- ExitMBB->transferSuccessorsAndUpdatePHIs(BB);
-
- MachineRegisterInfo &RegInfo = MF->getRegInfo();
- unsigned ScratchRes = RegInfo.createVirtualRegister(&ARM64::GPR32RegClass);
- unsigned ScratchLo = IncrLo, ScratchHi = IncrHi;
- if (BinOpcodeLo) {
- assert(BinOpcodeHi && "Expect neither or both opcodes to be defined");
- ScratchLo = RegInfo.createVirtualRegister(&ARM64::GPR64RegClass);
- ScratchHi = RegInfo.createVirtualRegister(&ARM64::GPR64RegClass);
- }
-
- // ThisMBB:
- // ...
- // fallthrough --> LoopMBB
- BB->addSuccessor(LoopMBB);
-
- // LoopMBB:
- // ldxp DestLo, DestHi, Ptr
- // <binoplo> ScratchLo, DestLo, IncrLo
- // <binophi> ScratchHi, DestHi, IncrHi
- // stxp ScratchRes, ScratchLo, ScratchHi, ptr
- // cbnz ScratchRes, LoopMBB
- // fallthrough --> ExitMBB
- BB = LoopMBB;
- BuildMI(BB, DL, TII->get(LdrOpc), DestLo)
- .addReg(DestHi, RegState::Define)
- .addReg(Ptr);
- if (BinOpcodeLo) {
- // operand order needs to go the other way for NAND
- if (BinOpcodeLo == ARM64::BICXrr) {
- std::swap(IncrLo, DestLo);
- std::swap(IncrHi, DestHi);
- }
-
- BuildMI(BB, DL, TII->get(BinOpcodeLo), ScratchLo).addReg(DestLo).addReg(
- IncrLo);
- BuildMI(BB, DL, TII->get(BinOpcodeHi), ScratchHi).addReg(DestHi).addReg(
- IncrHi);
- }
-
- BuildMI(BB, DL, TII->get(StrOpc), ScratchRes)
- .addReg(ScratchLo)
- .addReg(ScratchHi)
- .addReg(Ptr);
- BuildMI(BB, DL, TII->get(ARM64::CBNZW)).addReg(ScratchRes).addMBB(LoopMBB);
-
- BB->addSuccessor(LoopMBB);
- BB->addSuccessor(ExitMBB);
-
- // ExitMBB:
- // ...
- BB = ExitMBB;
-
- MI->eraseFromParent(); // The instruction is gone now.
-
- return BB;
-}
-
-MachineBasicBlock *
-ARM64TargetLowering::EmitAtomicCmpSwap128(MachineInstr *MI,
- MachineBasicBlock *BB) const {
- unsigned DestLo = MI->getOperand(0).getReg();
- unsigned DestHi = MI->getOperand(1).getReg();
- unsigned Ptr = MI->getOperand(2).getReg();
- unsigned OldValLo = MI->getOperand(3).getReg();
- unsigned OldValHi = MI->getOperand(4).getReg();
- unsigned NewValLo = MI->getOperand(5).getReg();
- unsigned NewValHi = MI->getOperand(6).getReg();
- AtomicOrdering Ord = static_cast<AtomicOrdering>(MI->getOperand(7).getImm());
- unsigned ScratchRes = BB->getParent()->getRegInfo().createVirtualRegister(
- &ARM64::GPR32RegClass);
- const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
- DebugLoc DL = MI->getDebugLoc();
-
- unsigned LdrOpc, StrOpc;
- getExclusiveOperation(16, Ord, LdrOpc, StrOpc);
-
- MachineFunction *MF = BB->getParent();
- const BasicBlock *LLVM_BB = BB->getBasicBlock();
- MachineFunction::iterator It = BB;
- ++It; // insert the new blocks after the current block
-
- MachineBasicBlock *Loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MachineBasicBlock *Loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MachineBasicBlock *ExitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MF->insert(It, Loop1MBB);
- MF->insert(It, Loop2MBB);
- MF->insert(It, ExitMBB);
-
- // Transfer the remainder of BB and its successor edges to exitMBB.
- ExitMBB->splice(ExitMBB->begin(), BB,
- std::next(MachineBasicBlock::iterator(MI)), BB->end());
- ExitMBB->transferSuccessorsAndUpdatePHIs(BB);
-
- // ThisMBB:
- // ...
- // fallthrough --> Loop1MBB
- BB->addSuccessor(Loop1MBB);
-
- // Loop1MBB:
- // ldxp DestLo, DestHi, [Ptr]
- // cmp DestLo, OldValLo
- // sbc xzr, DestHi, OldValHi
- // bne ExitMBB
- BB = Loop1MBB;
- BuildMI(BB, DL, TII->get(LdrOpc), DestLo)
- .addReg(DestHi, RegState::Define)
- .addReg(Ptr);
- BuildMI(BB, DL, TII->get(ARM64::SUBSXrr), ARM64::XZR).addReg(DestLo).addReg(
- OldValLo);
- BuildMI(BB, DL, TII->get(ARM64::SBCXr), ARM64::XZR).addReg(DestHi).addReg(
- OldValHi);
-
- BuildMI(BB, DL, TII->get(ARM64::Bcc)).addImm(ARM64CC::NE).addMBB(ExitMBB);
- BB->addSuccessor(Loop2MBB);
- BB->addSuccessor(ExitMBB);
-
- // Loop2MBB:
- // stxp ScratchRes, NewValLo, NewValHi, [Ptr]
- // cbnz ScratchRes, Loop1MBB
- BB = Loop2MBB;
- BuildMI(BB, DL, TII->get(StrOpc), ScratchRes)
- .addReg(NewValLo)
- .addReg(NewValHi)
- .addReg(Ptr);
- BuildMI(BB, DL, TII->get(ARM64::CBNZW)).addReg(ScratchRes).addMBB(Loop1MBB);
- BB->addSuccessor(Loop1MBB);
- BB->addSuccessor(ExitMBB);
-
- // ExitMBB:
- // ...
- BB = ExitMBB;
-
- MI->eraseFromParent(); // The instruction is gone now.
-
- return BB;
-}
-
-MachineBasicBlock *ARM64TargetLowering::EmitAtomicMinMax128(
- MachineInstr *MI, MachineBasicBlock *BB, unsigned CondCode) const {
- // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
- const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
-
- const BasicBlock *LLVM_BB = BB->getBasicBlock();
- MachineFunction *MF = BB->getParent();
- MachineFunction::iterator It = BB;
- ++It;
-
- unsigned DestLo = MI->getOperand(0).getReg();
- unsigned DestHi = MI->getOperand(1).getReg();
- unsigned Ptr = MI->getOperand(2).getReg();
- unsigned IncrLo = MI->getOperand(3).getReg();
- unsigned IncrHi = MI->getOperand(4).getReg();
- AtomicOrdering Ord = static_cast<AtomicOrdering>(MI->getOperand(5).getImm());
- DebugLoc DL = MI->getDebugLoc();
-
- unsigned LdrOpc, StrOpc;
- getExclusiveOperation(16, Ord, LdrOpc, StrOpc);
-
- MachineBasicBlock *LoopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MachineBasicBlock *ExitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MF->insert(It, LoopMBB);
- MF->insert(It, ExitMBB);
-
- // Transfer the remainder of BB and its successor edges to exitMBB.
- ExitMBB->splice(ExitMBB->begin(), BB,
- std::next(MachineBasicBlock::iterator(MI)), BB->end());
- ExitMBB->transferSuccessorsAndUpdatePHIs(BB);
-
- MachineRegisterInfo &RegInfo = MF->getRegInfo();
- unsigned ScratchRes = RegInfo.createVirtualRegister(&ARM64::GPR32RegClass);
- unsigned ScratchLo = RegInfo.createVirtualRegister(&ARM64::GPR64RegClass);
- unsigned ScratchHi = RegInfo.createVirtualRegister(&ARM64::GPR64RegClass);
-
- // ThisMBB:
- // ...
- // fallthrough --> LoopMBB
- BB->addSuccessor(LoopMBB);
-
- // LoopMBB:
- // ldxp DestLo, DestHi, Ptr
- // cmp ScratchLo, DestLo, IncrLo
- // sbc xzr, ScratchHi, DestHi, IncrHi
- // csel ScratchLo, DestLo, IncrLo, <cmp-op>
- // csel ScratchHi, DestHi, IncrHi, <cmp-op>
- // stxp ScratchRes, ScratchLo, ScratchHi, ptr
- // cbnz ScratchRes, LoopMBB
- // fallthrough --> ExitMBB
- BB = LoopMBB;
- BuildMI(BB, DL, TII->get(LdrOpc), DestLo)
- .addReg(DestHi, RegState::Define)
- .addReg(Ptr);
-
- BuildMI(BB, DL, TII->get(ARM64::SUBSXrr), ARM64::XZR).addReg(DestLo).addReg(
- IncrLo);
- BuildMI(BB, DL, TII->get(ARM64::SBCXr), ARM64::XZR).addReg(DestHi).addReg(
- IncrHi);
-
- BuildMI(BB, DL, TII->get(ARM64::CSELXr), ScratchLo)
- .addReg(DestLo)
- .addReg(IncrLo)
- .addImm(CondCode);
- BuildMI(BB, DL, TII->get(ARM64::CSELXr), ScratchHi)
- .addReg(DestHi)
- .addReg(IncrHi)
- .addImm(CondCode);
-
- BuildMI(BB, DL, TII->get(StrOpc), ScratchRes)
- .addReg(ScratchLo)
- .addReg(ScratchHi)
- .addReg(Ptr);
- BuildMI(BB, DL, TII->get(ARM64::CBNZW)).addReg(ScratchRes).addMBB(LoopMBB);
-
- BB->addSuccessor(LoopMBB);
- BB->addSuccessor(ExitMBB);
-
- // ExitMBB:
- // ...
- BB = ExitMBB;
-
- MI->eraseFromParent(); // The instruction is gone now.
-
- return BB;
-}
-