diff options
21 files changed, 2454 insertions, 40 deletions
diff --git a/lib/Target/AMDGPU/AMDGPU.h b/lib/Target/AMDGPU/AMDGPU.h index 568682899be..bbb542eb4fc 100644 --- a/lib/Target/AMDGPU/AMDGPU.h +++ b/lib/Target/AMDGPU/AMDGPU.h @@ -45,6 +45,7 @@ FunctionPass *createSILoadStoreOptimizerPass(); FunctionPass *createSIWholeQuadModePass(); FunctionPass *createSIFixControlFlowLiveIntervalsPass(); FunctionPass *createSIFixSGPRCopiesPass(); +FunctionPass *createSIMemoryLegalizerPass(); FunctionPass *createSIDebuggerInsertNopsPass(); FunctionPass *createSIInsertWaitsPass(); FunctionPass *createSIInsertWaitcntsPass(); @@ -125,6 +126,9 @@ extern char &AMDGPUCodeGenPrepareID; void initializeSIAnnotateControlFlowPass(PassRegistry&); extern char &SIAnnotateControlFlowPassID; +void initializeSIMemoryLegalizerPass(PassRegistry&); +extern char &SIMemoryLegalizerID; + void initializeSIDebuggerInsertNopsPass(PassRegistry&); extern char &SIDebuggerInsertNopsID; diff --git a/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp b/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp new file mode 100644 index 00000000000..3164140abe2 --- /dev/null +++ b/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp @@ -0,0 +1,29 @@ +//===--- AMDGPUMachineModuleInfo.cpp ----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file +/// \brief AMDGPU Machine Module Info. +/// +// +//===----------------------------------------------------------------------===// + +#include "AMDGPUMachineModuleInfo.h" +#include "llvm/IR/Module.h" + +namespace llvm { + +AMDGPUMachineModuleInfo::AMDGPUMachineModuleInfo(const MachineModuleInfo &MMI) + : MachineModuleInfoELF(MMI) { + LLVMContext &CTX = MMI.getModule()->getContext(); + AgentSSID = CTX.getOrInsertSyncScopeID("agent"); + WorkgroupSSID = CTX.getOrInsertSyncScopeID("workgroup"); + WavefrontSSID = CTX.getOrInsertSyncScopeID("wavefront"); +} + +} // end namespace llvm diff --git a/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h b/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h new file mode 100644 index 00000000000..98fe75b7a13 --- /dev/null +++ b/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h @@ -0,0 +1,57 @@ +//===--- AMDGPUMachineModuleInfo.h ------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file +/// \brief AMDGPU Machine Module Info. +/// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEMODULEINFO_H +#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEMODULEINFO_H + +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineModuleInfoImpls.h" +#include "llvm/IR/LLVMContext.h" + +namespace llvm { + +class AMDGPUMachineModuleInfo final : public MachineModuleInfoELF { +private: + + // All supported memory/synchronization scopes can be found here: + // http://llvm.org/docs/AMDGPUUsage.html#memory-scopes + + /// \brief Agent synchronization scope ID. + SyncScope::ID AgentSSID; + /// \brief Workgroup synchronization scope ID. + SyncScope::ID WorkgroupSSID; + /// \brief Wavefront synchronization scope ID. + SyncScope::ID WavefrontSSID; + +public: + AMDGPUMachineModuleInfo(const MachineModuleInfo &MMI); + + /// \returns Agent synchronization scope ID. + SyncScope::ID getAgentSSID() const { + return AgentSSID; + } + /// \returns Workgroup synchronization scope ID. + SyncScope::ID getWorkgroupSSID() const { + return WorkgroupSSID; + } + /// \returns Wavefront synchronization scope ID. + SyncScope::ID getWavefrontSSID() const { + return WavefrontSSID; + } +}; + +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEMODULEINFO_H diff --git a/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index dc868f010d8..1f8b6a88faa 100644 --- a/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -150,6 +150,7 @@ extern "C" void LLVMInitializeAMDGPUTarget() { initializeSIWholeQuadModePass(*PR); initializeSILowerControlFlowPass(*PR); initializeSIInsertSkipsPass(*PR); + initializeSIMemoryLegalizerPass(*PR); initializeSIDebuggerInsertNopsPass(*PR); initializeSIOptimizeExecMaskingPass(*PR); initializeAMDGPUUnifyDivergentExitNodesPass(*PR); @@ -806,6 +807,7 @@ void GCNPassConfig::addPreEmitPass() { addPass(createSIInsertWaitsPass()); addPass(createSIShrinkInstructionsPass()); addPass(&SIInsertSkipsPassID); + addPass(createSIMemoryLegalizerPass()); addPass(createSIDebuggerInsertNopsPass()); addPass(&BranchRelaxationPassID); } diff --git a/lib/Target/AMDGPU/BUFInstructions.td b/lib/Target/AMDGPU/BUFInstructions.td index 89caa65cdaf..cbf9d1fa700 100644 --- a/lib/Target/AMDGPU/BUFInstructions.td +++ b/lib/Target/AMDGPU/BUFInstructions.td @@ -1058,12 +1058,12 @@ multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Ins def : Pat < (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc))), - (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0) + (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) >; def : Pat < (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), - (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0) + (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) >; } @@ -1174,12 +1174,12 @@ multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo In def : Pat < (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), vt:$val), - (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0) + (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) >; def : Pat < (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), - (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0) + (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) >; } let Predicates = [isSICI] in { diff --git a/lib/Target/AMDGPU/CMakeLists.txt b/lib/Target/AMDGPU/CMakeLists.txt index 971208c5db8..164a57d848d 100644 --- a/lib/Target/AMDGPU/CMakeLists.txt +++ b/lib/Target/AMDGPU/CMakeLists.txt @@ -51,6 +51,7 @@ add_llvm_target(AMDGPUCodeGen AMDGPUMCInstLower.cpp AMDGPUMachineCFGStructurizer.cpp AMDGPUMachineFunction.cpp + AMDGPUMachineModuleInfo.cpp AMDGPUUnifyMetadata.cpp AMDGPUOpenCLImageTypeLoweringPass.cpp AMDGPUSubtarget.cpp @@ -93,6 +94,7 @@ add_llvm_target(AMDGPUCodeGen SILowerI1Copies.cpp SIMachineFunctionInfo.cpp SIMachineScheduler.cpp + SIMemoryLegalizer.cpp SIOptimizeExecMasking.cpp SIPeepholeSDWA.cpp SIRegisterInfo.cpp diff --git a/lib/Target/AMDGPU/FLATInstructions.td b/lib/Target/AMDGPU/FLATInstructions.td index b038cc4fa2c..8913b620c4f 100644 --- a/lib/Target/AMDGPU/FLATInstructions.td +++ b/lib/Target/AMDGPU/FLATInstructions.td @@ -620,7 +620,7 @@ class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : Pat class FlatLoadAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : Pat < (vt (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc))), - (inst $vaddr, $offset, 1, $slc) + (inst $vaddr, $offset, 0, $slc) >; class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : Pat < @@ -632,7 +632,7 @@ class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt // atomic store follows atomic binop convention so the address comes // first. (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), vt:$data), - (inst $vaddr, $data, $offset, 1, $slc) + (inst $vaddr, $data, $offset, 0, $slc) >; class FlatAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, diff --git a/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/lib/Target/AMDGPU/SIMemoryLegalizer.cpp new file mode 100644 index 00000000000..48812c84b20 --- /dev/null +++ b/lib/Target/AMDGPU/SIMemoryLegalizer.cpp @@ -0,0 +1,491 @@ +//===--- SIMemoryLegalizer.cpp ----------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file +/// \brief Memory legalizer - implements memory model. More information can be +/// found here: +/// http://llvm.org/docs/AMDGPUUsage.html#memory-model +/// +// +//===----------------------------------------------------------------------===// + +#include "AMDGPU.h" +#include "AMDGPUMachineModuleInfo.h" +#include "AMDGPUSubtarget.h" +#include "Utils/AMDGPUBaseInfo.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/IR/DiagnosticInfo.h" + +using namespace llvm; +using namespace llvm::AMDGPU; + +#define DEBUG_TYPE "si-memory-legalizer" +#define PASS_NAME "SI Memory Legalizer" + +namespace { + +class SIMemoryLegalizer final : public MachineFunctionPass { +private: + struct AtomicInfo final { + SyncScope::ID SSID = SyncScope::System; + AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent; + AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent; + + AtomicInfo() {} + + AtomicInfo(SyncScope::ID SSID, + AtomicOrdering Ordering, + AtomicOrdering FailureOrdering) + : SSID(SSID), + Ordering(Ordering), + FailureOrdering(FailureOrdering) {} + + AtomicInfo(const MachineMemOperand *MMO) + : SSID(MMO->getSyncScopeID()), + Ordering(MMO->getOrdering()), + FailureOrdering(MMO->getFailureOrdering()) {} + }; + + /// \brief LLVM context. + LLVMContext *CTX = nullptr; + /// \brief Machine module info. + const AMDGPUMachineModuleInfo *MMI = nullptr; + /// \brief Instruction info. + const SIInstrInfo *TII = nullptr; + + /// \brief Immediate for "vmcnt(0)". + unsigned Vmcnt0Immediate = 0; + /// \brief Opcode for cache invalidation instruction (L1). + unsigned Wbinvl1Opcode = 0; + + /// \brief List of atomic pseudo instructions. + std::list<MachineBasicBlock::iterator> AtomicPseudoMIs; + + /// \brief Inserts "buffer_wbinvl1_vol" instruction \p Before or after \p MI. + /// Always returns true. + bool insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI, + bool Before = true) const; + /// \brief Inserts "s_waitcnt vmcnt(0)" instruction \p Before or after \p MI. + /// Always returns true. + bool insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI, + bool Before = true) const; + + /// \brief Sets GLC bit if present in \p MI. Returns true if \p MI is + /// modified, false otherwise. + bool setGLC(const MachineBasicBlock::iterator &MI) const; + + /// \brief Removes all processed atomic pseudo instructions from the current + /// function. Returns true if current function is modified, false otherwise. + bool removeAtomicPseudoMIs(); + + /// \brief Reports unknown synchronization scope used in \p MI to LLVM + /// context. + void reportUnknownSynchScope(const MachineBasicBlock::iterator &MI); + + /// \returns Atomic fence info if \p MI is an atomic fence operation, + /// "None" otherwise. + Optional<AtomicInfo> getAtomicFenceInfo( + const MachineBasicBlock::iterator &MI) const; + /// \returns Atomic load info if \p MI is an atomic load operation, + /// "None" otherwise. + Optional<AtomicInfo> getAtomicLoadInfo( + const MachineBasicBlock::iterator &MI) const; + /// \returns Atomic store info if \p MI is an atomic store operation, + /// "None" otherwise. + Optional<AtomicInfo> getAtomicStoreInfo( + const MachineBasicBlock::iterator &MI) const; + /// \returns Atomic cmpxchg info if \p MI is an atomic cmpxchg operation, + /// "None" otherwise. + Optional<AtomicInfo> getAtomicCmpxchgInfo( + const MachineBasicBlock::iterator &MI) const; + /// \returns Atomic rmw info if \p MI is an atomic rmw operation, + /// "None" otherwise. + Optional<AtomicInfo> getAtomicRmwInfo( + const MachineBasicBlock::iterator &MI) const; + + /// \brief Expands atomic fence operation \p MI. Returns true if + /// instructions are added/deleted or \p MI is modified, false otherwise. + bool expandAtomicFence(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI); + /// \brief Expands atomic load operation \p MI. Returns true if + /// instructions are added/deleted or \p MI is modified, false otherwise. + bool expandAtomicLoad(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI); + /// \brief Expands atomic store operation \p MI. Returns true if + /// instructions are added/deleted or \p MI is modified, false otherwise. + bool expandAtomicStore(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI); + /// \brief Expands atomic cmpxchg operation \p MI. Returns true if + /// instructions are added/deleted or \p MI is modified, false otherwise. + bool expandAtomicCmpxchg(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI); + /// \brief Expands atomic rmw operation \p MI. Returns true if + /// instructions are added/deleted or \p MI is modified, false otherwise. + bool expandAtomicRmw(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI); + +public: + static char ID; + + SIMemoryLegalizer() + : MachineFunctionPass(ID) {} + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + StringRef getPassName() const override { + return PASS_NAME; + } + + bool runOnMachineFunction(MachineFunction &MF) override; +}; + +} // end namespace anonymous + +bool SIMemoryLegalizer::insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI, + bool Before) const { + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (!Before) + ++MI; + + BuildMI(MBB, MI, DL, TII->get(Wbinvl1Opcode)); + + if (!Before) + --MI; + + return true; +} + +bool SIMemoryLegalizer::insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI, + bool Before) const { + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (!Before) + ++MI; + + BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Vmcnt0Immediate); + + if (!Before) + --MI; + + return true; +} + +bool SIMemoryLegalizer::setGLC(const MachineBasicBlock::iterator &MI) const { + int GLCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::glc); + if (GLCIdx == -1) + return false; + + MachineOperand &GLC = MI->getOperand(GLCIdx); + if (GLC.getImm() == 1) + return false; + + GLC.setImm(1); + return true; +} + +bool SIMemoryLegalizer::removeAtomicPseudoMIs() { + if (AtomicPseudoMIs.empty()) + return false; + + for (auto &MI : AtomicPseudoMIs) + MI->eraseFromParent(); + + AtomicPseudoMIs.clear(); + return true; +} + +void SIMemoryLegalizer::reportUnknownSynchScope( + const MachineBasicBlock::iterator &MI) { + DiagnosticInfoUnsupported Diag(*MI->getParent()->getParent()->getFunction(), + "Unsupported synchronization scope"); + CTX->diagnose(Diag); +} + +Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicFenceInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE) + return None; + + SyncScope::ID SSID = + static_cast<SyncScope::ID>(MI->getOperand(1).getImm()); + AtomicOrdering Ordering = + static_cast<AtomicOrdering>(MI->getOperand(0).getImm()); + return AtomicInfo(SSID, Ordering, AtomicOrdering::NotAtomic); +} + +Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicLoadInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (!(MI->mayLoad() && !MI->mayStore())) + return None; + if (!MI->hasOneMemOperand()) + return AtomicInfo(); + + const MachineMemOperand *MMO = *MI->memoperands_begin(); + if (!MMO->isAtomic()) + return None; + + return AtomicInfo(MMO); +} + +Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicStoreInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (!(!MI->mayLoad() && MI->mayStore())) + return None; + if (!MI->hasOneMemOperand()) + return AtomicInfo(); + + const MachineMemOperand *MMO = *MI->memoperands_begin(); + if (!MMO->isAtomic()) + return None; + + return AtomicInfo(MMO); +} + +Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicCmpxchgInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (!(MI->mayLoad() && MI->mayStore())) + return None; + if (!MI->hasOneMemOperand()) + return AtomicInfo(); + + const MachineMemOperand *MMO = *MI->memoperands_begin(); + if (!MMO->isAtomic()) + return None; + if (MMO->getFailureOrdering() == AtomicOrdering::NotAtomic) + return None; + + return AtomicInfo(MMO); +} + +Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicRmwInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (!(MI->mayLoad() && MI->mayStore())) + return None; + if (!MI->hasOneMemOperand()) + return AtomicInfo(); + + const MachineMemOperand *MMO = *MI->memoperands_begin(); + if (!MMO->isAtomic()) + return None; + if (MMO->getFailureOrdering() != AtomicOrdering::NotAtomic) + return None; + + return AtomicInfo(MMO); +} + +bool SIMemoryLegalizer::expandAtomicFence(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI) { + assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE); + + bool Changed = false; + if (AI.SSID == SyncScope::System || + AI.SSID == MMI->getAgentSSID()) { + if (AI.Ordering == AtomicOrdering::Acquire || + AI.Ordering == AtomicOrdering::Release || + AI.Ordering == AtomicOrdering::AcquireRelease || + AI.Ordering == AtomicOrdering::SequentiallyConsistent) + Changed |= insertWaitcntVmcnt0(MI); + + if (AI.Ordering == AtomicOrdering::Acquire || + AI.Ordering == AtomicOrdering::AcquireRelease || + AI.Ordering == AtomicOrdering::SequentiallyConsistent) + Changed |= insertBufferWbinvl1Vol(MI); + + AtomicPseudoMIs.push_back(MI); + return Changed; + } else if (AI.SSID == SyncScope::SingleThread || + AI.SSID == MMI->getWorkgroupSSID() || + AI.SSID == MMI->getWavefrontSSID()) { + AtomicPseudoMIs.push_back(MI); + return Changed; + } else { + reportUnknownSynchScope(MI); + return Changed; + } +} + +bool SIMemoryLegalizer::expandAtomicLoad(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI) { + assert(MI->mayLoad() && !MI->mayStore()); + + bool Changed = false; + if (AI.SSID == SyncScope::System || + AI.SSID == MMI->getAgentSSID()) { + if (AI.Ordering == AtomicOrdering::Acquire || + AI.Ordering == AtomicOrdering::SequentiallyConsistent) + Changed |= setGLC(MI); + + if (AI.Ordering == AtomicOrdering::SequentiallyConsistent) + Changed |= insertWaitcntVmcnt0(MI); + + if (AI.Ordering == AtomicOrdering::Acquire || + AI.Ordering == AtomicOrdering::SequentiallyConsistent) { + Changed |= insertWaitcntVmcnt0(MI, false); + Changed |= insertBufferWbinvl1Vol(MI, false); + } + + return Changed; + } else if (AI.SSID == SyncScope::SingleThread || + AI.SSID == MMI->getWorkgroupSSID() || + AI.SSID == MMI->getWavefrontSSID()) { + return Changed; + } else { + reportUnknownSynchScope(MI); + return Changed; + } +} + +bool SIMemoryLegalizer::expandAtomicStore(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI) { + assert(!MI->mayLoad() && MI->mayStore()); + + bool Changed = false; + if (AI.SSID == SyncScope::System || + AI.SSID == MMI->getAgentSSID()) { + if (AI.Ordering == AtomicOrdering::Release || + AI.Ordering == AtomicOrdering::SequentiallyConsistent) + Changed |= insertWaitcntVmcnt0(MI); + + return Changed; + } else if (AI.SSID == SyncScope::SingleThread || + AI.SSID == MMI->getWorkgroupSSID() || + AI.SSID == MMI->getWavefrontSSID()) { + return Changed; + } else { + reportUnknownSynchScope(MI); + return Changed; + } +} + +bool SIMemoryLegalizer::expandAtomicCmpxchg(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI) { + assert(MI->mayLoad() && MI->mayStore()); + + bool Changed = false; + if (AI.SSID == SyncScope::System || + AI.SSID == MMI->getAgentSSID()) { + if (AI.Ordering == AtomicOrdering::Release || + AI.Ordering == AtomicOrdering::AcquireRelease || + AI.Ordering == AtomicOrdering::SequentiallyConsistent || + AI.FailureOrdering == AtomicOrdering::SequentiallyConsistent) + Changed |= insertWaitcntVmcnt0(MI); + + if (AI.Ordering == AtomicOrdering::Acquire || + AI.Ordering == AtomicOrdering::AcquireRelease || + AI.Ordering == AtomicOrdering::SequentiallyConsistent || + AI.FailureOrdering == AtomicOrdering::Acquire || + AI.FailureOrdering == AtomicOrdering::SequentiallyConsistent) { + Changed |= insertWaitcntVmcnt0(MI, false); + Changed |= insertBufferWbinvl1Vol(MI, false); + } + + return Changed; + } else if (AI.SSID == SyncScope::SingleThread || + AI.SSID == MMI->getWorkgroupSSID() || + AI.SSID == MMI->getWavefrontSSID()) { + Changed |= setGLC(MI); + return Changed; + } else { + reportUnknownSynchScope(MI); + return Changed; + } +} + +bool SIMemoryLegalizer::expandAtomicRmw(const AtomicInfo &AI, + MachineBasicBlock::iterator &MI) { + assert(MI->mayLoad() && MI->mayStore()); + + bool Changed = false; + if (AI.SSID == SyncScope::System || + AI.SSID == MMI->getAgentSSID()) { + if (AI.Ordering == AtomicOrdering::Release || + AI.Ordering == AtomicOrdering::AcquireRelease || + AI.Ordering == AtomicOrdering::SequentiallyConsistent) + Changed |= insertWaitcntVmcnt0(MI); + + if (AI.Ordering == AtomicOrdering::Acquire || + AI.Ordering == AtomicOrdering::AcquireRelease || + AI.Ordering == AtomicOrdering::SequentiallyConsistent) { + Changed |= insertWaitcntVmcnt0(MI, false); + Changed |= insertBufferWbinvl1Vol(MI, false); + } + + return Changed; + } else if (AI.SSID == SyncScope::SingleThread || + AI.SSID == MMI->getWorkgroupSSID() || + AI.SSID == MMI->getWavefrontSSID()) { + Changed |= setGLC(MI); + return Changed; + } else { + reportUnknownSynchScope(MI); + return Changed; + } +} + +bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) { + bool Changed = false; + const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); + const IsaInfo::IsaVersion IV = IsaInfo::getIsaVersion(ST.getFeatureBits()); + + CTX = &MF.getFunction()->getContext(); + MMI = &MF.getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>(); + TII = ST.getInstrInfo(); + + Vmcnt0Immediate = + AMDGPU::encodeWaitcnt(IV, 0, getExpcntBitMask(IV), getLgkmcntBitMask(IV)); + Wbinvl1Opcode = ST.getGeneration() <= AMDGPUSubtarget::SOUTHERN_ISLANDS ? + AMDGPU::BUFFER_WBINVL1 : AMDGPU::BUFFER_WBINVL1_VOL; + + for (auto &MBB : MF) { + for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) { + if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic)) + continue; + + if (const auto &AI = getAtomicFenceInfo(MI)) + Changed |= expandAtomicFence(AI.getValue(), MI); + else if (const auto &AI = getAtomicLoadInfo(MI)) + Changed |= expandAtomicLoad(AI.getValue(), MI); + else if (const auto &AI = getAtomicStoreInfo(MI)) + Changed |= expandAtomicStore(AI.getValue(), MI); + else if (const auto &AI = getAtomicCmpxchgInfo(MI)) + Changed |= expandAtomicCmpxchg(AI.getValue(), MI); + else if (const auto &AI = getAtomicRmwInfo(MI)) + Changed |= expandAtomicRmw(AI.getValue(), MI); + } + } + + Changed |= removeAtomicPseudoMIs(); + return Changed; +} + +INITIALIZE_PASS(SIMemoryLegalizer, DEBUG_TYPE, PASS_NAME, false, false) + +char SIMemoryLegalizer::ID = 0; +char &llvm::SIMemoryLegalizerID = SIMemoryLegalizer::ID; + +FunctionPass *llvm::createSIMemoryLegalizerPass() { + return new SIMemoryLegalizer(); +} diff --git a/test/CodeGen/AMDGPU/fence-amdgiz.ll b/test/CodeGen/AMDGPU/fence-amdgiz.ll index df675c9a869..98e9d2485d4 100644 --- a/test/CodeGen/AMDGPU/fence-amdgiz.ll +++ b/test/CodeGen/AMDGPU/fence-amdgiz.ll @@ -3,11 +3,12 @@ target datalayout = "e-p:64:64-p1:64:64-p2:64:64-p3:32:32-p4:32:32-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-A5" target triple = "amdgcn-amd-amdhsa-amdgizcl" -; CHECK_LABEL: atomic_fence -; CHECK: BB#0: -; CHECK: ATOMIC_FENCE 4, 1 -; CHECK: s_endpgm - +; CHECK-LABEL: atomic_fence +; CHECK: BB#0: +; CHECK-NOT: ATOMIC_FENCE +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: buffer_wbinvl1_vol +; CHECK-NEXT: s_endpgm define amdgpu_kernel void @atomic_fence() { fence acquire ret void diff --git a/test/CodeGen/AMDGPU/flat_atomics.ll b/test/CodeGen/AMDGPU/flat_atomics.ll index 8e153181dec..e3d307ea1ce 100644 --- a/test/CodeGen/AMDGPU/flat_atomics.ll +++ b/test/CodeGen/AMDGPU/flat_atomics.ll @@ -998,8 +998,8 @@ entry: } ; GCN-LABEL: {{^}}atomic_store_i32_offset: -; CIVI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} -; GFX9: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} offset:16 glc{{$}} +; CIVI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +; GFX9: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} offset:16{{$}} define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, i32 addrspace(4)* %out) { entry: %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 @@ -1008,7 +1008,7 @@ entry: } ; GCN-LABEL: {{^}}atomic_store_i32: -; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} define amdgpu_kernel void @atomic_store_i32(i32 %in, i32 addrspace(4)* %out) { entry: store atomic i32 %in, i32 addrspace(4)* %out seq_cst, align 4 @@ -1016,8 +1016,8 @@ entry: } ; GCN-LABEL: {{^}}atomic_store_i32_addr64_offset: -; CIVI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} -; GFX9: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} offset:16 glc{{$}} +; CIVI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +; GFX9: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} offset:16{{$}} define amdgpu_kernel void @atomic_store_i32_addr64_offset(i32 %in, i32 addrspace(4)* %out, i64 %index) { entry: %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index @@ -1027,7 +1027,7 @@ entry: } ; GCN-LABEL: {{^}}atomic_store_i32_addr64: -; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} define amdgpu_kernel void @atomic_store_i32_addr64(i32 %in, i32 addrspace(4)* %out, i64 %index) { entry: %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index diff --git a/test/CodeGen/AMDGPU/flat_atomics_i64.ll b/test/CodeGen/AMDGPU/flat_atomics_i64.ll index 723dde9ab68..acbfe6ab4cb 100644 --- a/test/CodeGen/AMDGPU/flat_atomics_i64.ll +++ b/test/CodeGen/AMDGPU/flat_atomics_i64.ll @@ -846,7 +846,7 @@ entry: } ; GCN-LABEL: {{^}}atomic_store_i64_offset: -; GCN: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, i64 addrspace(4)* %out) { entry: %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 @@ -855,7 +855,7 @@ entry: } ; GCN-LABEL: {{^}}atomic_store_i64: -; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}] glc +; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}] define amdgpu_kernel void @atomic_store_i64(i64 %in, i64 addrspace(4)* %out) { entry: store atomic i64 %in, i64 addrspace(4)* %out seq_cst, align 8 @@ -863,7 +863,7 @@ entry: } ; GCN-LABEL: {{^}}atomic_store_i64_addr64_offset: -; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}]{{$}} define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, i64 addrspace(4)* %out, i64 %index) { entry: %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index @@ -873,7 +873,7 @@ entry: } ; GCN-LABEL: {{^}}atomic_store_i64_addr64: -; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}]{{$}} define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, i64 addrspace(4)* %out, i64 %index) { entry: %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index diff --git a/test/CodeGen/AMDGPU/global_atomics.ll b/test/CodeGen/AMDGPU/global_atomics.ll index 6928bede547..7acceaf7e50 100644 --- a/test/CodeGen/AMDGPU/global_atomics.ll +++ b/test/CodeGen/AMDGPU/global_atomics.ll @@ -1004,8 +1004,8 @@ entry: } ; FUNC-LABEL: {{^}}atomic_store_i32_offset: -; SI: buffer_store_dword {{v[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}} -; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +; SI: buffer_store_dword {{v[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}} +; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, i32 addrspace(1)* %out) { entry: %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4 @@ -1014,8 +1014,8 @@ entry: } ; FUNC-LABEL: {{^}}atomic_store_i32: -; SI: buffer_store_dword {{v[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc{{$}} -; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +; SI: buffer_store_dword {{v[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}} +; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} define amdgpu_kernel void @atomic_store_i32(i32 %in, i32 addrspace(1)* %out) { entry: store atomic i32 %in, i32 addrspace(1)* %out seq_cst, align 4 @@ -1023,8 +1023,8 @@ entry: } ; FUNC-LABEL: {{^}}atomic_store_i32_addr64_offset: -; SI: buffer_store_dword {{v[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}} -; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +; SI: buffer_store_dword {{v[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}} +; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} define amdgpu_kernel void @atomic_store_i32_addr64_offset(i32 %in, i32 addrspace(1)* %out, i64 %index) { entry: %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index @@ -1034,8 +1034,8 @@ entry: } ; FUNC-LABEL: {{^}}atomic_store_i32_addr64: -; SI: buffer_store_dword {{v[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}} -; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +; SI: buffer_store_dword {{v[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}} +; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} define amdgpu_kernel void @atomic_store_i32_addr64(i32 %in, i32 addrspace(1)* %out, i64 %index) { entry: %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index diff --git a/test/CodeGen/AMDGPU/global_atomics_i64.ll b/test/CodeGen/AMDGPU/global_atomics_i64.ll index 56520b787ea..8fb47c5eb00 100644 --- a/test/CodeGen/AMDGPU/global_atomics_i64.ll +++ b/test/CodeGen/AMDGPU/global_atomics_i64.ll @@ -997,8 +997,8 @@ entry: } ; FUNC-LABEL: {{^}}atomic_store_i64_offset: -; CI: buffer_store_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}} -; VI: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; CI: buffer_store_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}} +; VI: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, i64 addrspace(1)* %out) { entry: %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4 @@ -1007,8 +1007,8 @@ entry: } ; FUNC-LABEL: {{^}}atomic_store_i64: -; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc -; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}] glc +; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}} +; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}]{{$}} define amdgpu_kernel void @atomic_store_i64(i64 %in, i64 addrspace(1)* %out) { entry: store atomic i64 %in, i64 addrspace(1)* %out seq_cst, align 8 @@ -1016,8 +1016,8 @@ entry: } ; FUNC-LABEL: {{^}}atomic_store_i64_addr64_offset: -; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}} -; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}} +; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}} +; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}]{{$}} define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, i64 addrspace(1)* %out, i64 %index) { entry: %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index @@ -1027,8 +1027,8 @@ entry: } ; FUNC-LABEL: {{^}}atomic_store_i64_addr64: -; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}} -; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}} +; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}} +; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}]{{$}} define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, i64 addrspace(1)* %out, i64 %index) { entry: %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index diff --git a/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll b/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll new file mode 100644 index 00000000000..15fff2ac0e1 --- /dev/null +++ b/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll @@ -0,0 +1,652 @@ +; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s + +; CHECK-LABEL: {{^}}system_monotonic_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_monotonic_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in monotonic monotonic + ret void +} + +; CHECK-LABEL: {{^}}system_acquire_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_acquire_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in acquire monotonic + ret void +} + +; CHECK-LABEL: {{^}}system_release_monotonic +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_release_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in release monotonic + ret void +} + +; CHECK-LABEL: {{^}}system_acq_rel_monotonic +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_acq_rel_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in acq_rel monotonic + ret void +} + +; CHECK-LABEL: {{^}}system_seq_cst_monotonic +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_seq_cst_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst monotonic + ret void +} + +; CHECK-LABEL: {{^}}system_acquire_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_acquire_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in acquire acquire + ret void +} + +; CHECK-LABEL: {{^}}system_release_acquire +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_release_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in release acquire + ret void +} + +; CHECK-LABEL: {{^}}system_acq_rel_acquire +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_acq_rel_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in acq_rel acquire + ret void +} + +; CHECK-LABEL: {{^}}system_seq_cst_acquire +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_seq_cst_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst acquire + ret void +} + +; CHECK-LABEL: {{^}}system_seq_cst_seq_cst +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_seq_cst_seq_cst( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst seq_cst + ret void +} + +; CHECK-LABEL: {{^}}singlethread_monotonic_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_monotonic_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic + ret void +} + +; CHECK-LABEL: {{^}}singlethread_acquire_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_acquire_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic + ret void +} + +; CHECK-LABEL: {{^}}singlethread_release_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_release_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic + ret void +} + +; CHECK-LABEL: {{^}}singlethread_acq_rel_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_acq_rel_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic + ret void +} + +; CHECK-LABEL: {{^}}singlethread_seq_cst_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_seq_cst_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic + ret void +} + +; CHECK-LABEL: {{^}}singlethread_acquire_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_acquire_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire + ret void +} + +; CHECK-LABEL: {{^}}singlethread_release_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_release_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire + ret void +} + +; CHECK-LABEL: {{^}}singlethread_acq_rel_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_acq_rel_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire + ret void +} + +; CHECK-LABEL: {{^}}singlethread_seq_cst_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_seq_cst_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire + ret void +} + +; CHECK-LABEL: {{^}}singlethread_seq_cst_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_seq_cst_seq_cst( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst + ret void +} + +; CHECK-LABEL: {{^}}agent_monotonic_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_monotonic_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic + ret void +} + +; CHECK-LABEL: {{^}}agent_acquire_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_acquire_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic + ret void +} + +; CHECK-LABEL: {{^}}agent_release_monotonic +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_release_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") release monotonic + ret void +} + +; CHECK-LABEL: {{^}}agent_acq_rel_monotonic +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_acq_rel_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic + ret void +} + +; CHECK-LABEL: {{^}}agent_seq_cst_monotonic +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_seq_cst_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic + ret void +} + +; CHECK-LABEL: {{^}}agent_acquire_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_acquire_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire + ret void +} + +; CHECK-LABEL: {{^}}agent_release_acquire +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_release_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") release acquire + ret void +} + +; CHECK-LABEL: {{^}}agent_acq_rel_acquire +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_acq_rel_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire + ret void +} + +; CHECK-LABEL: {{^}}agent_seq_cst_acquire +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_seq_cst_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire + ret void +} + +; CHECK-LABEL: {{^}}agent_seq_cst_seq_cst +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_seq_cst_seq_cst( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst + ret void +} + +; CHECK-LABEL: {{^}}workgroup_monotonic_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_monotonic_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic + ret void +} + +; CHECK-LABEL: {{^}}workgroup_acquire_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_acquire_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic + ret void +} + +; CHECK-LABEL: {{^}}workgroup_release_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_release_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic + ret void +} + +; CHECK-LABEL: {{^}}workgroup_acq_rel_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_acq_rel_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic + ret void +} + +; CHECK-LABEL: {{^}}workgroup_seq_cst_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_seq_cst_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic + ret void +} + +; CHECK-LABEL: {{^}}workgroup_acquire_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_acquire_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire + ret void +} + +; CHECK-LABEL: {{^}}workgroup_release_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_release_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire + ret void +} + +; CHECK-LABEL: {{^}}workgroup_acq_rel_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_acq_rel_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire + ret void +} + +; CHECK-LABEL: {{^}}workgroup_seq_cst_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_seq_cst_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire + ret void +} + +; CHECK-LABEL: {{^}}workgroup_seq_cst_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_seq_cst_seq_cst( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst + ret void +} + +; CHECK-LABEL: {{^}}wavefront_monotonic_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_monotonic_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic + ret void +} + +; CHECK-LABEL: {{^}}wavefront_acquire_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_acquire_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic + ret void +} + +; CHECK-LABEL: {{^}}wavefront_release_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_release_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic + ret void +} + +; CHECK-LABEL: {{^}}wavefront_acq_rel_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_acq_rel_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic + ret void +} + +; CHECK-LABEL: {{^}}wavefront_seq_cst_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_seq_cst_monotonic( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic + ret void +} + +; CHECK-LABEL: {{^}}wavefront_acquire_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_acquire_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire + ret void +} + +; CHECK-LABEL: {{^}}wavefront_release_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_release_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire + ret void +} + +; CHECK-LABEL: {{^}}wavefront_acq_rel_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_acq_rel_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire + ret void +} + +; CHECK-LABEL: {{^}}wavefront_seq_cst_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_seq_cst_acquire( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire + ret void +} + +; CHECK-LABEL: {{^}}wavefront_seq_cst_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_seq_cst_seq_cst( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst + ret void +} diff --git a/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll b/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll new file mode 100644 index 00000000000..a563cfd0283 --- /dev/null +++ b/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll @@ -0,0 +1,225 @@ +; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=GCN -check-prefix=GFX6 %s +; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=GCN -check-prefix=GFX8 %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=GCN -check-prefix=GFX8 %s + +; FUNC-LABEL: {{^}}system_acquire +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GFX6: s_waitcnt vmcnt(0){{$}} +; GFX6-NEXT: buffer_wbinvl1{{$}} +; GFX8: s_waitcnt vmcnt(0){{$}} +; GFX8-NEXT: buffer_wbinvl1_vol{{$}} +; GCN: s_endpgm +define amdgpu_kernel void @system_acquire() { +entry: + fence acquire + ret void +} + +; FUNC-LABEL: {{^}}system_release +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_waitcnt vmcnt(0){{$}} +; GCN: s_endpgm +define amdgpu_kernel void @system_release() { +entry: + fence release + ret void +} + +; FUNC-LABEL: {{^}}system_acq_rel +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_waitcnt vmcnt(0){{$}} +; GFX6: buffer_wbinvl1{{$}} +; GFX8: buffer_wbinvl1_vol{{$}} +; GCN: s_endpgm +define amdgpu_kernel void @system_acq_rel() { +entry: + fence acq_rel + ret void +} + +; FUNC-LABEL: {{^}}system_seq_cst +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_waitcnt vmcnt(0){{$}} +; GFX6: buffer_wbinvl1{{$}} +; GFX8: buffer_wbinvl1_vol{{$}} +; GCN: s_endpgm +define amdgpu_kernel void @system_seq_cst() { +entry: + fence seq_cst + ret void +} + +; FUNC-LABEL: {{^}}singlethread_acquire +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @singlethread_acquire() { +entry: + fence syncscope("singlethread") acquire + ret void +} + +; FUNC-LABEL: {{^}}singlethread_release +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @singlethread_release() { +entry: + fence syncscope("singlethread") release + ret void +} + +; FUNC-LABEL: {{^}}singlethread_acq_rel +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @singlethread_acq_rel() { +entry: + fence syncscope("singlethread") acq_rel + ret void +} + +; FUNC-LABEL: {{^}}singlethread_seq_cst +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @singlethread_seq_cst() { +entry: + fence syncscope("singlethread") seq_cst + ret void +} + +; FUNC-LABEL: {{^}}agent_acquire +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GFX6: s_waitcnt vmcnt(0){{$}} +; GFX6-NEXT: buffer_wbinvl1{{$}} +; GFX8: s_waitcnt vmcnt(0){{$}} +; GFX8-NEXT: buffer_wbinvl1_vol{{$}} +; GCN: s_endpgm +define amdgpu_kernel void @agent_acquire() { +entry: + fence syncscope("agent") acquire + ret void +} + +; FUNC-LABEL: {{^}}agent_release +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_waitcnt vmcnt(0){{$}} +; GCN: s_endpgm +define amdgpu_kernel void @agent_release() { +entry: + fence syncscope("agent") release + ret void +} + +; FUNC-LABEL: {{^}}agent_acq_rel +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_waitcnt vmcnt(0){{$}} +; GFX6: buffer_wbinvl1{{$}} +; GFX8: buffer_wbinvl1_vol{{$}} +; GCN: s_endpgm +define amdgpu_kernel void @agent_acq_rel() { +entry: + fence syncscope("agent") acq_rel + ret void +} + +; FUNC-LABEL: {{^}}agent_seq_cst +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_waitcnt vmcnt(0){{$}} +; GFX6: buffer_wbinvl1{{$}} +; GFX8: buffer_wbinvl1_vol{{$}} +; GCN: s_endpgm +define amdgpu_kernel void @agent_seq_cst() { +entry: + fence syncscope("agent") seq_cst + ret void +} + +; FUNC-LABEL: {{^}}workgroup_acquire +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @workgroup_acquire() { +entry: + fence syncscope("workgroup") acquire + ret void +} + +; FUNC-LABEL: {{^}}workgroup_release +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @workgroup_release() { +entry: + fence syncscope("workgroup") release + ret void +} + +; FUNC-LABEL: {{^}}workgroup_acq_rel +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @workgroup_acq_rel() { +entry: + fence syncscope("workgroup") acq_rel + ret void +} + +; FUNC-LABEL: {{^}}workgroup_seq_cst +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @workgroup_seq_cst() { +entry: + fence syncscope("workgroup") seq_cst + ret void +} + +; FUNC-LABEL: {{^}}wavefront_acquire +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @wavefront_acquire() { +entry: + fence syncscope("wavefront") acquire + ret void +} + +; FUNC-LABEL: {{^}}wavefront_release +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @wavefront_release() { +entry: + fence syncscope("wavefront") release + ret void +} + +; FUNC-LABEL: {{^}}wavefront_acq_rel +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @wavefront_acq_rel() { +entry: + fence syncscope("wavefront") acq_rel + ret void +} + +; FUNC-LABEL: {{^}}wavefront_seq_cst +; GCN: BB#0 +; GCN-NOT: ATOMIC_FENCE +; GCN: s_endpgm +define amdgpu_kernel void @wavefront_seq_cst() { +entry: + fence syncscope("wavefront") seq_cst + ret void +} diff --git a/test/CodeGen/AMDGPU/memory-legalizer-atomic-load.ll b/test/CodeGen/AMDGPU/memory-legalizer-atomic-load.ll new file mode 100644 index 00000000000..0434978caa2 --- /dev/null +++ b/test/CodeGen/AMDGPU/memory-legalizer-atomic-load.ll @@ -0,0 +1,282 @@ +; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s + +; CHECK-LABEL: {{^}}system_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @system_unordered( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in unordered, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}system_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @system_monotonic( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in monotonic, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}system_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @system_acquire( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in acquire, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}system_seq_cst +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @system_seq_cst( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}singlethread_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @singlethread_unordered( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("singlethread") unordered, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}singlethread_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @singlethread_monotonic( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("singlethread") monotonic, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}singlethread_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @singlethread_acquire( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("singlethread") acquire, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}singlethread_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @singlethread_seq_cst( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("singlethread") seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}agent_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @agent_unordered( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") unordered, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}agent_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @agent_monotonic( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") monotonic, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}agent_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @agent_acquire( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") acquire, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}agent_seq_cst +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @agent_seq_cst( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}workgroup_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @workgroup_unordered( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("workgroup") unordered, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}workgroup_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @workgroup_monotonic( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("workgroup") monotonic, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}workgroup_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @workgroup_acquire( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("workgroup") acquire, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}workgroup_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @workgroup_seq_cst( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("workgroup") seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}wavefront_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @wavefront_unordered( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("wavefront") unordered, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}wavefront_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @wavefront_monotonic( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("wavefront") monotonic, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}wavefront_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @wavefront_acquire( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("wavefront") acquire, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK-LABEL: {{^}}wavefront_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define amdgpu_kernel void @wavefront_seq_cst( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("wavefront") seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll b/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll new file mode 100644 index 00000000000..6f537c2014d --- /dev/null +++ b/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll @@ -0,0 +1,302 @@ +; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s + +; CHECK-LABEL: {{^}}system_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_monotonic( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in monotonic + ret void +} + +; CHECK-LABEL: {{^}}system_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_acquire( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in acquire + ret void +} + +; CHECK-LABEL: {{^}}system_release +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_release( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in release + ret void +} + +; CHECK-LABEL: {{^}}system_acq_rel +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_acq_rel( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in acq_rel + ret void +} + +; CHECK-LABEL: {{^}}system_seq_cst +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @system_seq_cst( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; CHECK-LABEL: {{^}}singlethread_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_monotonic( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") monotonic + ret void +} + +; CHECK-LABEL: {{^}}singlethread_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_acquire( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") acquire + ret void +} + +; CHECK-LABEL: {{^}}singlethread_release +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_release( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") release + ret void +} + +; CHECK-LABEL: {{^}}singlethread_acq_rel +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_acq_rel( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") acq_rel + ret void +} + +; CHECK-LABEL: {{^}}singlethread_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @singlethread_seq_cst( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") seq_cst + ret void +} + +; CHECK-LABEL: {{^}}agent_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_monotonic( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") monotonic + ret void +} + +; CHECK-LABEL: {{^}}agent_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_acquire( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") acquire + ret void +} + +; CHECK-LABEL: {{^}}agent_release +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_release( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") release + ret void +} + +; CHECK-LABEL: {{^}}agent_acq_rel +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_acq_rel( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") acq_rel + ret void +} + +; CHECK-LABEL: {{^}}agent_seq_cst +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NEXT: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: buffer_wbinvl1_vol +define amdgpu_kernel void @agent_seq_cst( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") seq_cst + ret void +} + +; CHECK-LABEL: {{^}}workgroup_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_monotonic( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") monotonic + ret void +} + +; CHECK-LABEL: {{^}}workgroup_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_acquire( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") acquire + ret void +} + +; CHECK-LABEL: {{^}}workgroup_release +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_release( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") release + ret void +} + +; CHECK-LABEL: {{^}}workgroup_acq_rel +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_acq_rel( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") acq_rel + ret void +} + +; CHECK-LABEL: {{^}}workgroup_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @workgroup_seq_cst( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") seq_cst + ret void +} + +; CHECK-LABEL: {{^}}wavefront_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_monotonic( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") monotonic + ret void +} + +; CHECK-LABEL: {{^}}wavefront_acquire +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_acquire( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") acquire + ret void +} + +; CHECK-LABEL: {{^}}wavefront_release +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_release( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") release + ret void +} + +; CHECK-LABEL: {{^}}wavefront_acq_rel +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_acq_rel( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") acq_rel + ret void +} + +; CHECK-LABEL: {{^}}wavefront_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK-NOT: buffer_wbinvl1_vol +define amdgpu_kernel void @wavefront_seq_cst( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") seq_cst + ret void +} diff --git a/test/CodeGen/AMDGPU/memory-legalizer-atomic-store.ll b/test/CodeGen/AMDGPU/memory-legalizer-atomic-store.ll new file mode 100644 index 00000000000..14b7d3e62f8 --- /dev/null +++ b/test/CodeGen/AMDGPU/memory-legalizer-atomic-store.ll @@ -0,0 +1,202 @@ +; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s + +; CHECK-LABEL: {{^}}system_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @system_unordered( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out unordered, align 4 + ret void +} + +; CHECK-LABEL: {{^}}system_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @system_monotonic( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out monotonic, align 4 + ret void +} + +; CHECK-LABEL: {{^}}system_release +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @system_release( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out release, align 4 + ret void +} + +; CHECK-LABEL: {{^}}system_seq_cst +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @system_seq_cst( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out seq_cst, align 4 + ret void +} + +; CHECK-LABEL: {{^}}singlethread_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @singlethread_unordered( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("singlethread") unordered, align 4 + ret void +} + +; CHECK-LABEL: {{^}}singlethread_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @singlethread_monotonic( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("singlethread") monotonic, align 4 + ret void +} + +; CHECK-LABEL: {{^}}singlethread_release +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @singlethread_release( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("singlethread") release, align 4 + ret void +} + +; CHECK-LABEL: {{^}}singlethread_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @singlethread_seq_cst( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("singlethread") seq_cst, align 4 + ret void +} + +; CHECK-LABEL: {{^}}agent_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @agent_unordered( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("agent") unordered, align 4 + ret void +} + +; CHECK-LABEL: {{^}}agent_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @agent_monotonic( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("agent") monotonic, align 4 + ret void +} + +; CHECK-LABEL: {{^}}agent_release +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @agent_release( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("agent") release, align 4 + ret void +} + +; CHECK-LABEL: {{^}}agent_seq_cst +; CHECK: s_waitcnt vmcnt(0){{$}} +; CHECK-NEXT: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @agent_seq_cst( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("agent") seq_cst, align 4 + ret void +} + +; CHECK-LABEL: {{^}}workgroup_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @workgroup_unordered( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("workgroup") unordered, align 4 + ret void +} + +; CHECK-LABEL: {{^}}workgroup_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @workgroup_monotonic( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("workgroup") monotonic, align 4 + ret void +} + +; CHECK-LABEL: {{^}}workgroup_release +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @workgroup_release( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("workgroup") release, align 4 + ret void +} + +; CHECK-LABEL: {{^}}workgroup_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @workgroup_seq_cst( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("workgroup") seq_cst, align 4 + ret void +} + +; CHECK-LABEL: {{^}}wavefront_unordered +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @wavefront_unordered( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("wavefront") unordered, align 4 + ret void +} + +; CHECK-LABEL: {{^}}wavefront_monotonic +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @wavefront_monotonic( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("wavefront") monotonic, align 4 + ret void +} + +; CHECK-LABEL: {{^}}wavefront_release +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @wavefront_release( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("wavefront") release, align 4 + ret void +} + +; CHECK-LABEL: {{^}}wavefront_seq_cst +; CHECK-NOT: s_waitcnt vmcnt(0){{$}} +; CHECK: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}} +define amdgpu_kernel void @wavefront_seq_cst( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("wavefront") seq_cst, align 4 + ret void +} diff --git a/test/CodeGen/AMDGPU/memory-legalizer-invalid-syncscope.ll b/test/CodeGen/AMDGPU/memory-legalizer-invalid-syncscope.ll new file mode 100644 index 00000000000..b89bb0f7866 --- /dev/null +++ b/test/CodeGen/AMDGPU/memory-legalizer-invalid-syncscope.ll @@ -0,0 +1,43 @@ +; RUN: not llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s 2>&1 | FileCheck %s +; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s 2>&1 | FileCheck %s + +; CHECK: error: <unknown>:0:0: in function invalid_fence void (): Unsupported synchronization scope +define amdgpu_kernel void @invalid_fence() { +entry: + fence syncscope("invalid") seq_cst + ret void +} + +; CHECK: error: <unknown>:0:0: in function invalid_load void (i32 addrspace(4)*, i32 addrspace(4)*): Unsupported synchronization scope +define amdgpu_kernel void @invalid_load( + i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in syncscope("invalid") seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; CHECK: error: <unknown>:0:0: in function invalid_store void (i32, i32 addrspace(4)*): Unsupported synchronization scope +define amdgpu_kernel void @invalid_store( + i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out syncscope("invalid") seq_cst, align 4 + ret void +} + +; CHECK: error: <unknown>:0:0: in function invalid_cmpxchg void (i32 addrspace(4)*, i32, i32): Unsupported synchronization scope +define amdgpu_kernel void @invalid_cmpxchg( + i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("invalid") seq_cst seq_cst + ret void +} + +; CHECK: error: <unknown>:0:0: in function invalid_rmw void (i32 addrspace(4)*, i32): Unsupported synchronization scope +define amdgpu_kernel void @invalid_rmw( + i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("invalid") seq_cst + ret void +} diff --git a/test/CodeGen/AMDGPU/syncscopes.ll b/test/CodeGen/AMDGPU/syncscopes.ll index 3741ce78899..6e356f69e05 100644 --- a/test/CodeGen/AMDGPU/syncscopes.ll +++ b/test/CodeGen/AMDGPU/syncscopes.ll @@ -1,9 +1,9 @@ ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -stop-before=si-debugger-insert-nops < %s | FileCheck --check-prefix=GCN %s ; GCN-LABEL: name: syncscopes -; GCN: FLAT_STORE_DWORD killed %vgpr1_vgpr2, killed %vgpr0, 0, -1, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("agent") seq_cst 4 into %ir.agent_out) -; GCN: FLAT_STORE_DWORD killed %vgpr4_vgpr5, killed %vgpr3, 0, -1, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("workgroup") seq_cst 4 into %ir.workgroup_out) -; GCN: FLAT_STORE_DWORD killed %vgpr7_vgpr8, killed %vgpr6, 0, -1, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("wavefront") seq_cst 4 into %ir.wavefront_out) +; GCN: FLAT_STORE_DWORD killed %vgpr1_vgpr2, killed %vgpr0, 0, 0, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("agent") seq_cst 4 into %ir.agent_out) +; GCN: FLAT_STORE_DWORD killed %vgpr4_vgpr5, killed %vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("workgroup") seq_cst 4 into %ir.workgroup_out) +; GCN: FLAT_STORE_DWORD killed %vgpr7_vgpr8, killed %vgpr6, 0, 0, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("wavefront") seq_cst 4 into %ir.wavefront_out) define void @syncscopes( i32 %agent, i32 addrspace(4)* %agent_out, diff --git a/test/CodeGen/MIR/AMDGPU/memory-legalizer-atomic-insert-end.mir b/test/CodeGen/MIR/AMDGPU/memory-legalizer-atomic-insert-end.mir new file mode 100644 index 00000000000..d4ddfbe31b9 --- /dev/null +++ b/test/CodeGen/MIR/AMDGPU/memory-legalizer-atomic-insert-end.mir @@ -0,0 +1,122 @@ +# RUN: llc -march=amdgcn -mcpu=gfx803 -run-pass si-memory-legalizer %s -o - | FileCheck %s + +--- | + ; ModuleID = '<stdin>' + source_filename = "<stdin>" + target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64" + + ; Function Attrs: nounwind readnone + declare i32 @llvm.amdgcn.workitem.id.x() #0 + + ; Function Attrs: nounwind + define amdgpu_kernel void @atomic_max_i32_noret( + i32 addrspace(1)* %out, + i32 addrspace(1)* addrspace(1)* %in, + i32 addrspace(1)* %x, + i32 %y) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %idxprom = sext i32 %tid to i64 + %tid.gep = getelementptr i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %in, i64 %idxprom + %ptr = load volatile i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %tid.gep + %xor = xor i32 %tid, 1 + %cmp = icmp ne i32 %xor, 0 + %1 = call { i1, i64 } @llvm.amdgcn.if(i1 %cmp) + %2 = extractvalue { i1, i64 } %1, 0 + %3 = extractvalue { i1, i64 } %1, 1 + br i1 %2, label %atomic, label %exit + + atomic: ; preds = %0 + %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 100 + %ret = atomicrmw max i32 addrspace(1)* %gep, i32 %y seq_cst + br label %exit + + exit: ; preds = %atomic, %0 + call void @llvm.amdgcn.end.cf(i64 %3) + ret void + } + + declare { i1, i64 } @llvm.amdgcn.if(i1) + + declare void @llvm.amdgcn.end.cf(i64) + + ; Function Attrs: nounwind + declare void @llvm.stackprotector(i8*, i8**) #3 + + attributes #0 = { nounwind readnone "target-cpu"="tahiti" } + attributes #1 = { nounwind "target-cpu"="tahiti" } + attributes #2 = { readnone } + attributes #3 = { nounwind } + +... +--- + +# CHECK-LABEL: name: atomic_max_i32_noret + +# CHECK-LABEL: bb.1.atomic: +# CHECK: BUFFER_ATOMIC_SMAX_ADDR64 +# CHECK-NEXT: S_WAITCNT 3952 +# CHECK-NEXT: BUFFER_WBINVL1_VOL + +name: atomic_max_i32_noret +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%sgpr0_sgpr1' } + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0 (%ir-block.0): + successors: %bb.1.atomic(0x40000000), %bb.2.exit(0x40000000) + liveins: %vgpr0, %sgpr0_sgpr1 + + %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM %sgpr0_sgpr1, 11, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`) + %vgpr1 = V_ASHRREV_I32_e32 31, %vgpr0, implicit %exec + %vgpr1_vgpr2 = V_LSHL_B64 %vgpr0_vgpr1, 3, implicit %exec + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 0 + S_WAITCNT 127 + %vgpr1_vgpr2 = BUFFER_LOAD_DWORDX2_ADDR64 killed %vgpr1_vgpr2, %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 8 from %ir.tid.gep) + %vgpr0 = V_XOR_B32_e32 1, killed %vgpr0, implicit %exec + V_CMP_NE_U32_e32 0, killed %vgpr0, implicit-def %vcc, implicit %exec + %sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed %vcc, implicit-def %exec, implicit-def %scc, implicit %exec + %sgpr2_sgpr3 = S_XOR_B64 %exec, killed %sgpr2_sgpr3, implicit-def dead %scc + SI_MASK_BRANCH %bb.2.exit, implicit %exec + + bb.1.atomic: + successors: %bb.2.exit(0x80000000) + liveins: %sgpr4_sgpr5_sgpr6_sgpr7:0x0000000C, %sgpr0_sgpr1, %sgpr2_sgpr3, %vgpr1_vgpr2_vgpr3_vgpr4:0x00000003 + + %sgpr0 = S_LOAD_DWORD_IMM killed %sgpr0_sgpr1, 15, 0 :: (non-temporal dereferenceable invariant load 4 from `i32 addrspace(2)* undef`) + dead %vgpr0 = V_MOV_B32_e32 -1, implicit %exec + dead %vgpr0 = V_MOV_B32_e32 61440, implicit %exec + %sgpr4_sgpr5 = S_MOV_B64 0 + S_WAITCNT 127 + %vgpr0 = V_MOV_B32_e32 killed %sgpr0, implicit %exec, implicit %exec + S_WAITCNT 3952 + BUFFER_ATOMIC_SMAX_ADDR64 killed %vgpr0, killed %vgpr1_vgpr2, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 400, 0, implicit %exec :: (volatile load seq_cst 4 from %ir.gep) + + bb.2.exit: + liveins: %sgpr2_sgpr3 + + %exec = S_OR_B64 %exec, killed %sgpr2_sgpr3, implicit-def %scc + S_ENDPGM + +... + |