[NVPTX] added match.{any,all}.sync instructions, intrinsics & builtins.

Differential Revision: https://reviews.llvm.org/D38191

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@314223 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 93 insertions, 0 deletions

diff --git a/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 7d507100d3e..2f389860d14 100644
--- a/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -715,6 +715,10 @@ bool NVPTXDAGToDAGISel::tryIntrinsicNoChain(SDNode *N) {
   case Intrinsic::nvvm_texsurf_handle_internal:
     SelectTexSurfHandle(N);
     return true;
+  case Intrinsic::nvvm_match_all_sync_i32p:
+  case Intrinsic::nvvm_match_all_sync_i64p:
+    SelectMatchAll(N);
+    return true;
   }
 }
 
@@ -726,6 +730,36 @@ void NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
                                         MVT::i64, GlobalVal));
 }
 
+void NVPTXDAGToDAGISel::SelectMatchAll(SDNode *N) {
+  SDLoc DL(N);
+  enum { IS_I64 = 4, HAS_CONST_VALUE = 2, HAS_CONST_MASK = 1 };
+  unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
+  unsigned OpcodeIndex =
+      (IID == Intrinsic::nvvm_match_all_sync_i64p) ? IS_I64 : 0;
+  SDValue MaskOp = N->getOperand(1);
+  SDValue ValueOp = N->getOperand(2);
+  if (ConstantSDNode *ValueConst = dyn_cast<ConstantSDNode>(ValueOp)) {
+    OpcodeIndex |= HAS_CONST_VALUE;
+    ValueOp = CurDAG->getTargetConstant(ValueConst->getZExtValue(), DL,
+                                        ValueConst->getValueType(0));
+  }
+  if (ConstantSDNode *MaskConst = dyn_cast<ConstantSDNode>(MaskOp)) {
+    OpcodeIndex |= HAS_CONST_MASK;
+    MaskOp = CurDAG->getTargetConstant(MaskConst->getZExtValue(), DL,
+                                       MaskConst->getValueType(0));
+  }
+  // Maps {IS_I64, HAS_CONST_VALUE, HAS_CONST_MASK} -> opcode
+  unsigned Opcodes[8] = {
+      NVPTX::MATCH_ALLP_SYNC_32rr, NVPTX::MATCH_ALLP_SYNC_32ri,
+      NVPTX::MATCH_ALLP_SYNC_32ir, NVPTX::MATCH_ALLP_SYNC_32ii,
+      NVPTX::MATCH_ALLP_SYNC_64rr, NVPTX::MATCH_ALLP_SYNC_64ri,
+      NVPTX::MATCH_ALLP_SYNC_64ir, NVPTX::MATCH_ALLP_SYNC_64ii};
+  SDNode *NewNode = CurDAG->getMachineNode(Opcodes[OpcodeIndex], DL,
+                                           {ValueOp->getValueType(0), MVT::i1},
+                                           {MaskOp, ValueOp});
+  ReplaceNode(N, NewNode);
+}
+
 void NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
   SDValue Src = N->getOperand(0);
   AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
diff --git a/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index 8fc38e7c461..3ce7843b72f 100644
--- a/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -58,6 +58,7 @@ private:
   bool tryIntrinsicNoChain(SDNode *N);
   bool tryIntrinsicChain(SDNode *N);
   void SelectTexSurfHandle(SDNode *N);
+  void SelectMatchAll(SDNode *N);
   bool tryLoad(SDNode *N);
   bool tryLoadVector(SDNode *N);
   bool tryLDGLDU(SDNode *N);
diff --git a/lib/Target/NVPTX/NVPTXInstrInfo.td b/lib/Target/NVPTX/NVPTXInstrInfo.td
index 0a4fb0ed33f..92152a64e52 100644
--- a/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -158,6 +158,7 @@ def hasPTX31 : Predicate<"Subtarget->getPTXVersion() >= 31">;
 def hasPTX60 : Predicate<"Subtarget->getPTXVersion() >= 60">;
 
 def hasSM30 : Predicate<"Subtarget->getSmVersion() >= 30">;
+def hasSM70 : Predicate<"Subtarget->getSmVersion() >= 70">;
 
 def useFP16Math: Predicate<"Subtarget->allowFP16Math()">;
 
diff --git a/lib/Target/NVPTX/NVPTXIntrinsics.td b/lib/Target/NVPTX/NVPTXIntrinsics.td
index baa008f5494..11ebaaa5407 100644
--- a/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -247,6 +247,63 @@ defm VOTE_SYNC_ANY : VOTE_SYNC<Int1Regs, "any.pred", int_nvvm_vote_any_sync>;
 defm VOTE_SYNC_UNI : VOTE_SYNC<Int1Regs, "uni.pred", int_nvvm_vote_uni_sync>;
 defm VOTE_SYNC_BALLOT : VOTE_SYNC<Int32Regs, "ballot.b32", int_nvvm_vote_ballot_sync>;
 
+multiclass MATCH_ANY_SYNC<NVPTXRegClass regclass, string ptxtype, Intrinsic IntOp,
+                          Operand ImmOp> {
+  def ii : NVPTXInst<(outs regclass:$dest), (ins i32imm:$mask, ImmOp:$value),
+              "match.any.sync." # ptxtype # " \t$dest, $value, $mask;",
+              [(set regclass:$dest, (IntOp imm:$mask, imm:$value))]>,
+           Requires<[hasPTX60, hasSM70]>;
+  def ir : NVPTXInst<(outs regclass:$dest), (ins Int32Regs:$mask, ImmOp:$value),
+              "match.any.sync." # ptxtype # " \t$dest, $value, $mask;",
+              [(set regclass:$dest, (IntOp Int32Regs:$mask, imm:$value))]>,
+           Requires<[hasPTX60, hasSM70]>;
+  def ri : NVPTXInst<(outs regclass:$dest), (ins i32imm:$mask, regclass:$value),
+              "match.any.sync." # ptxtype # " \t$dest, $value, $mask;",
+              [(set regclass:$dest, (IntOp imm:$mask, regclass:$value))]>,
+           Requires<[hasPTX60, hasSM70]>;
+  def rr : NVPTXInst<(outs regclass:$dest), (ins Int32Regs:$mask, regclass:$value),
+              "match.any.sync." # ptxtype # " \t$dest, $value, $mask;",
+              [(set regclass:$dest, (IntOp Int32Regs:$mask, regclass:$value))]>,
+           Requires<[hasPTX60, hasSM70]>;
+}
+
+defm MATCH_ANY_SYNC_32 : MATCH_ANY_SYNC<Int32Regs, "b32", int_nvvm_match_any_sync_i32,
+                                        i32imm>;
+defm MATCH_ANY_SYNC_64 : MATCH_ANY_SYNC<Int64Regs, "b64", int_nvvm_match_any_sync_i64,
+                                        i64imm>;
+
+multiclass MATCH_ALLP_SYNC<NVPTXRegClass regclass, string ptxtype, Intrinsic IntOp,
+                          Operand ImmOp> {
+  def ii : NVPTXInst<(outs regclass:$dest, Int1Regs:$pred),
+                     (ins i32imm:$mask, ImmOp:$value),
+              "match.all.sync." # ptxtype # " \t$dest|$pred, $value, $mask;",
+              // If would be nice if tablegen could match multiple return values,
+              // but it does not seem to be the case. Thus we have an empty pattern and
+              // lower intrinsic to instruction manually.
+              // [(set regclass:$dest, Int1Regs:$pred, (IntOp imm:$value, imm:$mask))]>,
+              []>,
+           Requires<[hasPTX60, hasSM70]>;
+  def ir : NVPTXInst<(outs regclass:$dest, Int1Regs:$pred),
+                     (ins Int32Regs:$mask, ImmOp:$value),
+              "match.all.sync." # ptxtype # " \t$dest|$pred, $value, $mask;",
+              []>,
+           Requires<[hasPTX60, hasSM70]>;
+  def ri : NVPTXInst<(outs regclass:$dest, Int1Regs:$pred),
+                     (ins i32imm:$mask, regclass:$value),
+              "match.all.sync." # ptxtype # " \t$dest|$pred, $value, $mask;",
+              []>,
+           Requires<[hasPTX60, hasSM70]>;
+  def rr : NVPTXInst<(outs regclass:$dest, Int1Regs:$pred),
+                     (ins Int32Regs:$mask, regclass:$value),
+              "match.all.sync." # ptxtype # " \t$dest|$pred, $value, $mask;",
+              []>,
+           Requires<[hasPTX60, hasSM70]>;
+}
+defm MATCH_ALLP_SYNC_32 : MATCH_ALLP_SYNC<Int32Regs, "b32", int_nvvm_match_all_sync_i32p,
+                                         i32imm>;
+defm MATCH_ALLP_SYNC_64 : MATCH_ALLP_SYNC<Int64Regs, "b64", int_nvvm_match_all_sync_i64p,
+                                         i64imm>;
+
 } // isConvergent = 1
 
 //-----------------------------------