diff options
-rw-r--r-- | include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h | 2 | ||||
-rw-r--r-- | include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h | 24 | ||||
-rw-r--r-- | include/llvm/CodeGen/TargetOpcodes.def | 6 | ||||
-rw-r--r-- | include/llvm/Target/GenericOpcodes.td | 22 | ||||
-rw-r--r-- | include/llvm/Target/GlobalISel/SelectionDAGCompat.td | 22 | ||||
-rw-r--r-- | lib/CodeGen/GlobalISel/IRTranslator.cpp | 21 | ||||
-rw-r--r-- | lib/CodeGen/GlobalISel/MachineIRBuilder.cpp | 32 | ||||
-rw-r--r-- | lib/Target/AArch64/AArch64InstructionSelector.cpp | 6 | ||||
-rw-r--r-- | lib/Target/AArch64/AArch64LegalizerInfo.cpp | 8 | ||||
-rw-r--r-- | lib/Target/ARM/ARMInstructionSelector.cpp | 6 | ||||
-rw-r--r-- | lib/Target/X86/X86InstructionSelector.cpp | 5 | ||||
-rw-r--r-- | test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll | 20 | ||||
-rw-r--r-- | test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll | 12 | ||||
-rw-r--r-- | test/TableGen/GlobalISelEmitter.td | 2 | ||||
-rw-r--r-- | utils/TableGen/GlobalISelEmitter.cpp | 131 |
15 files changed, 220 insertions, 99 deletions
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index ac2c055ab14..b0859567db4 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -277,7 +277,7 @@ bool InstructionSelector::executeMatchTable( return false; for (const auto &MMO : State.MIs[InsnID]->memoperands()) - if (!isStrongerThan(Ordering, MMO->getOrdering())) + if (isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) if (handleReject() == RejectAndGiveUp) return false; break; diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index bed7c724892..1ea9dba5e99 100644 --- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -571,6 +571,30 @@ public: MachineInstrBuilder buildStore(unsigned Val, unsigned Addr, MachineMemOperand &MMO); + /// Build and insert `Res<def> = G_ATOMIC_LOAD Addr, MMO`. + /// + /// Loads the value stored at \p Addr. Puts the result in \p Res. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicLoad(unsigned Res, unsigned Addr, + MachineMemOperand &MMO); + + /// Build and insert `G_ATOMIC_STORE Val, Addr, MMO`. + /// + /// Stores the value \p Val to \p Addr. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Val must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicStore(unsigned Val, unsigned Addr, + MachineMemOperand &MMO); + /// Build and insert `Res0<def>, ... = G_EXTRACT Src, Idx0`. /// /// \pre setBasicBlock or setMI must have been called. diff --git a/include/llvm/CodeGen/TargetOpcodes.def b/include/llvm/CodeGen/TargetOpcodes.def index d3e8483798a..3cca0eb567e 100644 --- a/include/llvm/CodeGen/TargetOpcodes.def +++ b/include/llvm/CodeGen/TargetOpcodes.def @@ -265,6 +265,12 @@ HANDLE_TARGET_OPCODE(G_LOAD) /// Generic store. HANDLE_TARGET_OPCODE(G_STORE) +/// Generic atomic load +HANDLE_TARGET_OPCODE(G_ATOMIC_LOAD) + +/// Generic atomic store +HANDLE_TARGET_OPCODE(G_ATOMIC_STORE) + /// Generic atomic cmpxchg with internal success check. HANDLE_TARGET_OPCODE(G_ATOMIC_CMPXCHG_WITH_SUCCESS) diff --git a/include/llvm/Target/GenericOpcodes.td b/include/llvm/Target/GenericOpcodes.td index 489b569d990..76e672a37af 100644 --- a/include/llvm/Target/GenericOpcodes.td +++ b/include/llvm/Target/GenericOpcodes.td @@ -482,6 +482,28 @@ def G_STORE : Instruction { let mayStore = 1; } +// Generic atomic load. Expects a MachineMemOperand in addition to explicit +// operands. Technically, we could have handled this as a G_LOAD, however we +// decided to keep it separate on the basis that atomic loads tend to have +// very different handling to non-atomic loads. +def G_ATOMIC_LOAD : Instruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins ptype1:$addr); + let hasSideEffects = 0; + let mayLoad = 1; +} + +// Generic atomic store. Expects a MachineMemOperand in addition to explicit +// operands. Technically, we could have handled this as a G_STORE, however we +// decided to keep it separate on the basis that atomic stores tend to have +// very different handling to non-atomic stores. +def G_ATOMIC_STORE : Instruction { + let OutOperandList = (outs); + let InOperandList = (ins type0:$src, ptype1:$addr); + let hasSideEffects = 0; + let mayStore = 1; +} + // Generic atomic cmpxchg with internal success check. Expects a // MachineMemOperand in addition to explicit operands. def G_ATOMIC_CMPXCHG_WITH_SUCCESS : Instruction { diff --git a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 575f228cd77..71c8ce6d20c 100644 --- a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -23,11 +23,6 @@ class GINodeEquiv<Instruction i, SDNode node> { Instruction I = i; SDNode Node = node; - - // SelectionDAG has separate nodes for atomic and non-atomic memory operations - // (ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE) but GlobalISel - // stores this information in the MachineMemoryOperand. - bit CheckMMOIsNonAtomic = 0; } // These are defined in the same order as the G_* instructions. @@ -80,19 +75,16 @@ def : GINodeEquiv<G_BSWAP, bswap>; // Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some // complications that tablegen must take care of. For example, Predicates such // as isSignExtLoad require that this is not a perfect 1:1 mapping since a -// sign-extending load is (G_SEXT (G_LOAD x)) in GlobalISel. Additionally, -// G_LOAD handles both atomic and non-atomic loads where as SelectionDAG had -// separate nodes for them. This GINodeEquiv maps the non-atomic loads to -// G_LOAD with a non-atomic MachineMemOperand. -def : GINodeEquiv<G_LOAD, ld> { let CheckMMOIsNonAtomic = 1; } +// sign-extending load is (G_SEXT (G_LOAD x)) in GlobalISel. +def : GINodeEquiv<G_LOAD, ld>; // Broadly speaking G_STORE is equivalent to ISD::STORE but there are some // complications that tablegen must take care of. For example, predicates such // as isTruncStore require that this is not a perfect 1:1 mapping since a -// truncating store is (G_STORE (G_TRUNCATE x)) in GlobalISel. Additionally, -// G_STORE handles both atomic and non-atomic stores where as SelectionDAG had -// separate nodes for them. This GINodeEquiv maps the non-atomic stores to -// G_STORE with a non-atomic MachineMemOperand. -def : GINodeEquiv<G_STORE, st> { let CheckMMOIsNonAtomic = 1; } +// truncating store is (G_STORE (G_TRUNCATE x)) in GlobalISel. +def : GINodeEquiv<G_STORE, st>; + +def : GINodeEquiv<G_ATOMIC_LOAD, atomic_load>; +def : GINodeEquiv<G_ATOMIC_STORE, atomic_store>; def : GINodeEquiv<G_ATOMIC_CMPXCHG, atomic_cmp_swap>; def : GINodeEquiv<G_ATOMICRMW_XCHG, atomic_swap>; diff --git a/lib/CodeGen/GlobalISel/IRTranslator.cpp b/lib/CodeGen/GlobalISel/IRTranslator.cpp index e911085d0ad..774ea9877a7 100644 --- a/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -345,6 +345,16 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) { unsigned Res = getOrCreateVReg(LI); unsigned Addr = getOrCreateVReg(*LI.getPointerOperand()); + if (LI.getOrdering() != AtomicOrdering::NotAtomic) { + MIRBuilder.buildAtomicLoad( + Res, Addr, + *MF->getMachineMemOperand(MachinePointerInfo(LI.getPointerOperand()), + Flags, DL->getTypeStoreSize(LI.getType()), + getMemOpAlignment(LI), AAMDNodes(), nullptr, + LI.getSyncScopeID(), LI.getOrdering())); + return true; + } + MIRBuilder.buildLoad( Res, Addr, *MF->getMachineMemOperand(MachinePointerInfo(LI.getPointerOperand()), @@ -366,6 +376,17 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) { unsigned Val = getOrCreateVReg(*SI.getValueOperand()); unsigned Addr = getOrCreateVReg(*SI.getPointerOperand()); + if (SI.getOrdering() != AtomicOrdering::NotAtomic) { + MIRBuilder.buildAtomicStore( + Val, Addr, + *MF->getMachineMemOperand( + MachinePointerInfo(SI.getPointerOperand()), Flags, + DL->getTypeStoreSize(SI.getValueOperand()->getType()), + getMemOpAlignment(SI), AAMDNodes(), nullptr, SI.getSyncScopeID(), + SI.getOrdering())); + return true; + } + MIRBuilder.buildStore( Val, Addr, *MF->getMachineMemOperand( diff --git a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 62c396e6cdf..fbcb14d5252 100644 --- a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -295,6 +295,8 @@ MachineInstrBuilder MachineIRBuilder::buildLoad(unsigned Res, unsigned Addr, MachineMemOperand &MMO) { assert(MRI->getType(Res).isValid() && "invalid operand type"); assert(MRI->getType(Addr).isPointer() && "invalid operand type"); + assert(MMO.getOrdering() == AtomicOrdering::NotAtomic && + "invalid atomic ordering"); return buildInstr(TargetOpcode::G_LOAD) .addDef(Res) @@ -306,6 +308,8 @@ MachineInstrBuilder MachineIRBuilder::buildStore(unsigned Val, unsigned Addr, MachineMemOperand &MMO) { assert(MRI->getType(Val).isValid() && "invalid operand type"); assert(MRI->getType(Addr).isPointer() && "invalid operand type"); + assert(MMO.getOrdering() == AtomicOrdering::NotAtomic && + "invalid atomic ordering"); return buildInstr(TargetOpcode::G_STORE) .addUse(Val) @@ -313,6 +317,34 @@ MachineInstrBuilder MachineIRBuilder::buildStore(unsigned Val, unsigned Addr, .addMemOperand(&MMO); } +MachineInstrBuilder MachineIRBuilder::buildAtomicLoad(unsigned Res, + unsigned Addr, + MachineMemOperand &MMO) { + assert(MRI->getType(Res).isValid() && "invalid operand type"); + assert(MRI->getType(Addr).isPointer() && "invalid operand type"); + assert(MMO.getOrdering() != AtomicOrdering::NotAtomic && + "invalid atomic ordering"); + + return buildInstr(TargetOpcode::G_ATOMIC_LOAD) + .addDef(Res) + .addUse(Addr) + .addMemOperand(&MMO); +} + +MachineInstrBuilder MachineIRBuilder::buildAtomicStore(unsigned Val, + unsigned Addr, + MachineMemOperand &MMO) { + assert(MRI->getType(Val).isValid() && "invalid operand type"); + assert(MRI->getType(Addr).isPointer() && "invalid operand type"); + assert(MMO.getOrdering() != AtomicOrdering::NotAtomic && + "invalid atomic ordering"); + + return buildInstr(TargetOpcode::G_ATOMIC_STORE) + .addUse(Val) + .addUse(Addr) + .addMemOperand(&MMO); +} + MachineInstrBuilder MachineIRBuilder::buildUAdde(unsigned Res, unsigned CarryOut, unsigned Op0, unsigned Op1, diff --git a/lib/Target/AArch64/AArch64InstructionSelector.cpp b/lib/Target/AArch64/AArch64InstructionSelector.cpp index c2d3ae31c62..6999721b626 100644 --- a/lib/Target/AArch64/AArch64InstructionSelector.cpp +++ b/lib/Target/AArch64/AArch64InstructionSelector.cpp @@ -889,12 +889,6 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return false; } - auto &MemOp = **I.memoperands_begin(); - if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) { - DEBUG(dbgs() << "Atomic load/store not supported yet\n"); - return false; - } - const unsigned PtrReg = I.getOperand(1).getReg(); #ifndef NDEBUG const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI); diff --git a/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/lib/Target/AArch64/AArch64LegalizerInfo.cpp index 05df5120222..6531d5ebe4c 100644 --- a/lib/Target/AArch64/AArch64LegalizerInfo.cpp +++ b/lib/Target/AArch64/AArch64LegalizerInfo.cpp @@ -231,6 +231,14 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { setAction({MemOp, 1, p0}, Legal); } + for (unsigned MemOp : {G_ATOMIC_LOAD, G_ATOMIC_STORE}) { + for (auto Ty : {s8, s16, s32, s64, p0}) + setAction({MemOp, Ty}, Legal); + + // And everything's fine in addrspace 0. + setAction({MemOp, 1, p0}, Legal); + } + // Constants for (auto Ty : {s32, s64}) { setAction({TargetOpcode::G_CONSTANT, Ty}, Legal); diff --git a/lib/Target/ARM/ARMInstructionSelector.cpp b/lib/Target/ARM/ARMInstructionSelector.cpp index 6bbeae2e115..b43faf4903a 100644 --- a/lib/Target/ARM/ARMInstructionSelector.cpp +++ b/lib/Target/ARM/ARMInstructionSelector.cpp @@ -801,12 +801,6 @@ bool ARMInstructionSelector::select(MachineInstr &I, return selectGlobal(MIB, MRI); case G_STORE: case G_LOAD: { - const auto &MemOp = **I.memoperands_begin(); - if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) { - DEBUG(dbgs() << "Atomic load/store not supported yet\n"); - return false; - } - unsigned Reg = I.getOperand(0).getReg(); unsigned RegBank = RBI.getRegBank(Reg, MRI, TRI)->getID(); diff --git a/lib/Target/X86/X86InstructionSelector.cpp b/lib/Target/X86/X86InstructionSelector.cpp index 44bbc3f1b3f..f6530c4eee2 100644 --- a/lib/Target/X86/X86InstructionSelector.cpp +++ b/lib/Target/X86/X86InstructionSelector.cpp @@ -484,11 +484,6 @@ bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I, const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); auto &MemOp = **I.memoperands_begin(); - if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) { - DEBUG(dbgs() << "Atomic load/store not supported yet\n"); - return false; - } - unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment()); if (NewOpc == Opc) return false; diff --git a/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll index aa81c3aff89..afa003c4dd9 100644 --- a/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll +++ b/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll @@ -40,10 +40,10 @@ define [1 x double] @constant() { ret [1 x double] [double 1.0] } - ; The key problem here is that we may fail to create an MBB referenced by a - ; PHI. If so, we cannot complete the G_PHI and mustn't try or bad things - ; happen. -; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: cannot select: G_STORE %6, %2; mem:ST4[%addr] GPR:%6,%2 (in function: pending_phis) +; The key problem here is that we may fail to create an MBB referenced by a +; PHI. If so, we cannot complete the G_PHI and mustn't try or bad things +; happen. +; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate constant: [1 x double] (in function: pending_phis) ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for pending_phis ; FALLBACK-WITH-REPORT-OUT-LABEL: pending_phis: define i32 @pending_phis(i1 %tst, i32 %val, i32* %addr) { @@ -54,7 +54,7 @@ end: ret i32 %res true: - store atomic i32 42, i32* %addr seq_cst, align 4 + %t = extractvalue [1 x double] [double 1.0], 0 br label %end false: @@ -90,16 +90,6 @@ define i128 @sequence_sizes([8 x i8] %in) { ret i128 undef } -; Just to make sure we don't accidentally emit a normal load/store. -; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: cannot select: %2<def>(s64) = G_LOAD %0; mem:LD8[%addr] GPR:%2,%0 (in function: atomic_ops) -; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for atomic_ops -; FALLBACK-WITH-REPORT-LABEL: atomic_ops: -define i64 @atomic_ops(i64* %addr) { - store atomic i64 0, i64* %addr unordered, align 8 - %res = load atomic i64, i64* %addr seq_cst, align 8 - ret i64 %res -} - ; Make sure we don't mess up metadata arguments. declare void @llvm.write_register.i64(metadata, i64) diff --git a/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll index e7868327975..62203213a78 100644 --- a/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ b/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -1332,12 +1332,12 @@ define void @test_lifetime_intrin() { define void @test_load_store_atomics(i8* %addr) { ; CHECK-LABEL: name: test_load_store_atomics ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY %x0 -; CHECK: [[V0:%[0-9]+]]:_(s8) = G_LOAD [[ADDR]](p0) :: (load unordered 1 from %ir.addr) -; CHECK: G_STORE [[V0]](s8), [[ADDR]](p0) :: (store monotonic 1 into %ir.addr) -; CHECK: [[V1:%[0-9]+]]:_(s8) = G_LOAD [[ADDR]](p0) :: (load acquire 1 from %ir.addr) -; CHECK: G_STORE [[V1]](s8), [[ADDR]](p0) :: (store release 1 into %ir.addr) -; CHECK: [[V2:%[0-9]+]]:_(s8) = G_LOAD [[ADDR]](p0) :: (load syncscope("singlethread") seq_cst 1 from %ir.addr) -; CHECK: G_STORE [[V2]](s8), [[ADDR]](p0) :: (store syncscope("singlethread") monotonic 1 into %ir.addr) +; CHECK: [[V0:%[0-9]+]]:_(s8) = G_ATOMIC_LOAD [[ADDR]](p0) :: (load unordered 1 from %ir.addr) +; CHECK: G_ATOMIC_STORE [[V0]](s8), [[ADDR]](p0) :: (store monotonic 1 into %ir.addr) +; CHECK: [[V1:%[0-9]+]]:_(s8) = G_ATOMIC_LOAD [[ADDR]](p0) :: (load acquire 1 from %ir.addr) +; CHECK: G_ATOMIC_STORE [[V1]](s8), [[ADDR]](p0) :: (store release 1 into %ir.addr) +; CHECK: [[V2:%[0-9]+]]:_(s8) = G_ATOMIC_LOAD [[ADDR]](p0) :: (load syncscope("singlethread") seq_cst 1 from %ir.addr) +; CHECK: G_ATOMIC_STORE [[V2]](s8), [[ADDR]](p0) :: (store syncscope("singlethread") monotonic 1 into %ir.addr) %v0 = load atomic i8, i8* %addr unordered, align 1 store atomic i8 %v0, i8* %addr monotonic, align 1 diff --git a/test/TableGen/GlobalISelEmitter.td b/test/TableGen/GlobalISelEmitter.td index bdfda0a3be9..8bd31074af4 100644 --- a/test/TableGen/GlobalISelEmitter.td +++ b/test/TableGen/GlobalISelEmitter.td @@ -832,7 +832,6 @@ def MOVfpimmz : I<(outs FPR32:$dst), (ins f32imm:$imm), [(set FPR32:$dst, fpimmz // CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 22*/ [[LABEL:[0-9]+]], // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, -// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, @@ -861,7 +860,6 @@ def LOAD : I<(outs GPR32:$dst), (ins GPR32:$src1), // CHECK-NEXT: // MIs[0] Operand 1 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s16, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_LOAD, -// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/1, /*Order*/(int64_t)AtomicOrdering::NotAtomic, // CHECK-NEXT: // MIs[1] Operand 0 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s16, // CHECK-NEXT: // MIs[1] src1 diff --git a/utils/TableGen/GlobalISelEmitter.cpp b/utils/TableGen/GlobalISelEmitter.cpp index 062fe59d820..288f78b5d0c 100644 --- a/utils/TableGen/GlobalISelEmitter.cpp +++ b/utils/TableGen/GlobalISelEmitter.cpp @@ -2378,8 +2378,9 @@ private: CodeGenRegBank CGRegs; /// Keep track of the equivalence between SDNodes and Instruction by mapping - /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to - /// check for attributes on the relation such as CheckMMOIsNonAtomic. + /// SDNodes to the GINodeEquiv mapping. We map to the GINodeEquiv in case we + /// need to check for attributes on the relation such as (the now removed) + /// CheckMMOIsNonAtomic. /// This is defined using 'GINodeEquiv' in the target description. DenseMap<Record *, Record *> NodeEquivs; @@ -2398,6 +2399,8 @@ private: Record *findNodeEquiv(Record *N) const; Error importRulePredicates(RuleMatcher &M, ArrayRef<Predicate> Predicates); + Error importInstructionPredicates(InstructionMatcher &InsnMatcher, + const TreePatternNode *Src) const; Expected<InstructionMatcher &> createAndImportSelDAGMatcher( RuleMatcher &Rule, InstructionMatcher &InsnMatcher, const TreePatternNode *Src, unsigned &TempOpIdx) const; @@ -2483,45 +2486,8 @@ GlobalISelEmitter::importRulePredicates(RuleMatcher &M, return Error::success(); } -Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( - RuleMatcher &Rule, InstructionMatcher &InsnMatcher, - const TreePatternNode *Src, unsigned &TempOpIdx) const { - Record *SrcGIEquivOrNull = nullptr; - const CodeGenInstruction *SrcGIOrNull = nullptr; - - // Start with the defined operands (i.e., the results of the root operator). - if (Src->getExtTypes().size() > 1) - return failedImport("Src pattern has multiple results"); - - if (Src->isLeaf()) { - Init *SrcInit = Src->getLeafValue(); - if (isa<IntInit>(SrcInit)) { - InsnMatcher.addPredicate<InstructionOpcodeMatcher>( - &Target.getInstruction(RK.getDef("G_CONSTANT"))); - } else - return failedImport( - "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); - } else { - SrcGIEquivOrNull = findNodeEquiv(Src->getOperator()); - if (!SrcGIEquivOrNull) - return failedImport("Pattern operator lacks an equivalent Instruction" + - explainOperator(Src->getOperator())); - SrcGIOrNull = &Target.getInstruction(SrcGIEquivOrNull->getValueAsDef("I")); - - // The operators look good: match the opcode - InsnMatcher.addPredicate<InstructionOpcodeMatcher>(SrcGIOrNull); - } - - unsigned OpIdx = 0; - for (const TypeSetByHwMode &VTy : Src->getExtTypes()) { - // Results don't have a name unless they are the root node. The caller will - // set the name if appropriate. - OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx); - if (auto Error = OM.addTypeCheckPredicate(VTy, false /* OperandIsAPointer */)) - return failedImport(toString(std::move(Error)) + - " for result of Src pattern operator"); - } - +Error GlobalISelEmitter::importInstructionPredicates( + InstructionMatcher &InsnMatcher, const TreePatternNode *Src) const { for (const auto &Predicate : Src->getPredicateFns()) { if (Predicate.isAlwaysTrue()) continue; @@ -2610,9 +2576,50 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( return failedImport("Src pattern child has predicate (" + explainPredicates(Src) + ")"); } - if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) - InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic"); + return Error::success(); +} + +Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( + RuleMatcher &Rule, InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, unsigned &TempOpIdx) const { + Record *SrcGIEquivOrNull = nullptr; + const CodeGenInstruction *SrcGIOrNull = nullptr; + + // Start with the defined operands (i.e., the results of the root operator). + if (Src->getExtTypes().size() > 1) + return failedImport("Src pattern has multiple results"); + + if (Src->isLeaf()) { + Init *SrcInit = Src->getLeafValue(); + if (isa<IntInit>(SrcInit)) { + InsnMatcher.addPredicate<InstructionOpcodeMatcher>( + &Target.getInstruction(RK.getDef("G_CONSTANT"))); + } else + return failedImport( + "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); + } else { + SrcGIEquivOrNull = findNodeEquiv(Src->getOperator()); + if (!SrcGIEquivOrNull) + return failedImport("Pattern operator lacks an equivalent Instruction" + + explainOperator(Src->getOperator())); + SrcGIOrNull = &Target.getInstruction(SrcGIEquivOrNull->getValueAsDef("I")); + + // The operators look good: match the opcode + InsnMatcher.addPredicate<InstructionOpcodeMatcher>(SrcGIOrNull); + } + + unsigned OpIdx = 0; + for (const TypeSetByHwMode &VTy : Src->getExtTypes()) { + // Results don't have a name unless they are the root node. The caller will + // set the name if appropriate. + OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx); + if (auto Error = OM.addTypeCheckPredicate(VTy, false /* OperandIsAPointer */)) + return failedImport(toString(std::move(Error)) + + " for result of Src pattern operator"); + } + + if (Src->isLeaf()) { Init *SrcInit = Src->getLeafValue(); if (IntInit *SrcIntInit = dyn_cast<IntInit>(SrcInit)) { @@ -2631,6 +2638,8 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( // here since we don't support ImmLeaf predicates yet. However, we still // need to note the hidden operand to get GIM_CheckNumOperands correct. InsnMatcher.addOperand(OpIdx++, "", TempOpIdx); + if (auto Error = importInstructionPredicates(InsnMatcher, Src)) + return std::move(Error); return InsnMatcher; } @@ -2665,6 +2674,8 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( } } + if (auto Error = importInstructionPredicates(InsnMatcher, Src)) + return std::move(Error); return InsnMatcher; } @@ -3698,6 +3709,40 @@ TreePatternNode *GlobalISelEmitter::fixupPatternNode(TreePatternNode *N) { return Ext; } } + + if (N->getOperator()->getName() == "atomic_load") { + // If it's a atomic-load we need to adapt the pattern slightly. We need + // to split the node into (anyext (atomic_load ...)), and then apply the + // <<atomic_load_TY>> predicate by updating the result type of the load. + // + // For example: + // (atomic_load:[i32] [iPTR])<<atomic_load_i16>> + // must be transformed into: + // (anyext:[i32] (atomic_load:[i16] [iPTR])) + + std::vector<TreePredicateFn> Predicates; + Record *MemVT = nullptr; + for (const auto &P : N->getPredicateFns()) { + if (P.isAtomic() && P.getMemoryVT()) { + MemVT = P.getMemoryVT(); + continue; + } + Predicates.push_back(P); + } + + if (MemVT) { + TypeSetByHwMode ValueTy = getValueType(MemVT); + if (ValueTy != N->getType(0)) { + TreePatternNode *Ext = + new TreePatternNode(RK.getDef("anyext"), {N}, 1); + Ext->setType(0, N->getType(0)); + N->clearPredicateFns(); + N->setPredicateFns(Predicates); + N->setType(0, ValueTy); + return Ext; + } + } + } } return N; |