5 files changed, 60 insertions, 16 deletions
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
index ac2c055ab14..b0859567db4 100644
--- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
+++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
@@ -277,7 +277,7 @@ bool InstructionSelector::executeMatchTable(
           return false;
 
       for (const auto &MMO : State.MIs[InsnID]->memoperands())
-        if (!isStrongerThan(Ordering, MMO->getOrdering()))
+        if (isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
           if (handleReject() == RejectAndGiveUp)
             return false;
       break;
diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index bed7c724892..1ea9dba5e99 100644
--- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -571,6 +571,30 @@ public:
   MachineInstrBuilder buildStore(unsigned Val, unsigned Addr,
                                  MachineMemOperand &MMO);
 
+  /// Build and insert `Res<def> = G_ATOMIC_LOAD Addr, MMO`.
+  ///
+  /// Loads the value stored at \p Addr. Puts the result in \p Res.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Res must be a generic virtual register.
+  /// \pre \p Addr must be a generic virtual register with pointer type.
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildAtomicLoad(unsigned Res, unsigned Addr,
+                                      MachineMemOperand &MMO);
+
+  /// Build and insert `G_ATOMIC_STORE Val, Addr, MMO`.
+  ///
+  /// Stores the value \p Val to \p Addr.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Val must be a generic virtual register.
+  /// \pre \p Addr must be a generic virtual register with pointer type.
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildAtomicStore(unsigned Val, unsigned Addr,
+                                       MachineMemOperand &MMO);
+
   /// Build and insert `Res0<def>, ... = G_EXTRACT Src, Idx0`.
   ///
   /// \pre setBasicBlock or setMI must have been called.
diff --git a/include/llvm/CodeGen/TargetOpcodes.def b/include/llvm/CodeGen/TargetOpcodes.def
index d3e8483798a..3cca0eb567e 100644
--- a/include/llvm/CodeGen/TargetOpcodes.def
+++ b/include/llvm/CodeGen/TargetOpcodes.def
@@ -265,6 +265,12 @@ HANDLE_TARGET_OPCODE(G_LOAD)
 /// Generic store.
 HANDLE_TARGET_OPCODE(G_STORE)
 
+/// Generic atomic load
+HANDLE_TARGET_OPCODE(G_ATOMIC_LOAD)
+
+/// Generic atomic store
+HANDLE_TARGET_OPCODE(G_ATOMIC_STORE)
+
 /// Generic atomic cmpxchg with internal success check.
 HANDLE_TARGET_OPCODE(G_ATOMIC_CMPXCHG_WITH_SUCCESS)
 
diff --git a/include/llvm/Target/GenericOpcodes.td b/include/llvm/Target/GenericOpcodes.td
index 489b569d990..76e672a37af 100644
--- a/include/llvm/Target/GenericOpcodes.td
+++ b/include/llvm/Target/GenericOpcodes.td
@@ -482,6 +482,28 @@ def G_STORE : Instruction {
   let mayStore = 1;
 }
 
+// Generic atomic load. Expects a MachineMemOperand in addition to explicit
+// operands. Technically, we could have handled this as a G_LOAD, however we
+// decided to keep it separate on the basis that atomic loads tend to have
+// very different handling to non-atomic loads.
+def G_ATOMIC_LOAD : Instruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins ptype1:$addr);
+  let hasSideEffects = 0;
+  let mayLoad = 1;
+}
+
+// Generic atomic store. Expects a MachineMemOperand in addition to explicit
+// operands. Technically, we could have handled this as a G_STORE, however we
+// decided to keep it separate on the basis that atomic stores tend to have
+// very different handling to non-atomic stores.
+def G_ATOMIC_STORE : Instruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins type0:$src, ptype1:$addr);
+  let hasSideEffects = 0;
+  let mayStore = 1;
+}
+
 // Generic atomic cmpxchg with internal success check. Expects a
 // MachineMemOperand in addition to explicit operands.
 def G_ATOMIC_CMPXCHG_WITH_SUCCESS : Instruction {
diff --git a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 575f228cd77..71c8ce6d20c 100644
--- a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -23,11 +23,6 @@
 class GINodeEquiv<Instruction i, SDNode node> {
   Instruction I = i;
   SDNode Node = node;
-
-  // SelectionDAG has separate nodes for atomic and non-atomic memory operations
-  // (ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE) but GlobalISel
-  // stores this information in the MachineMemoryOperand.
-  bit CheckMMOIsNonAtomic = 0;
 }
 
 // These are defined in the same order as the G_* instructions.
@@ -80,19 +75,16 @@ def : GINodeEquiv<G_BSWAP, bswap>;
 // Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some
 // complications that tablegen must take care of. For example, Predicates such
 // as isSignExtLoad require that this is not a perfect 1:1 mapping since a
-// sign-extending load is (G_SEXT (G_LOAD x)) in GlobalISel. Additionally,
-// G_LOAD handles both atomic and non-atomic loads where as SelectionDAG had
-// separate nodes for them. This GINodeEquiv maps the non-atomic loads to
-// G_LOAD with a non-atomic MachineMemOperand.
-def : GINodeEquiv<G_LOAD, ld> { let CheckMMOIsNonAtomic = 1; }
+// sign-extending load is (G_SEXT (G_LOAD x)) in GlobalISel.
+def : GINodeEquiv<G_LOAD, ld>;
 // Broadly speaking G_STORE is equivalent to ISD::STORE but there are some
 // complications that tablegen must take care of. For example, predicates such
 // as isTruncStore require that this is not a perfect 1:1 mapping since a
-// truncating store is (G_STORE (G_TRUNCATE x)) in GlobalISel. Additionally,
-// G_STORE handles both atomic and non-atomic stores where as SelectionDAG had
-// separate nodes for them. This GINodeEquiv maps the non-atomic stores to
-// G_STORE with a non-atomic MachineMemOperand.
-def : GINodeEquiv<G_STORE, st> { let CheckMMOIsNonAtomic = 1; }
+// truncating store is (G_STORE (G_TRUNCATE x)) in GlobalISel.
+def : GINodeEquiv<G_STORE, st>;
+
+def : GINodeEquiv<G_ATOMIC_LOAD, atomic_load>;
+def : GINodeEquiv<G_ATOMIC_STORE, atomic_store>;
 
 def : GINodeEquiv<G_ATOMIC_CMPXCHG, atomic_cmp_swap>;
 def : GINodeEquiv<G_ATOMICRMW_XCHG, atomic_swap>;