diff options
author | Pierre van Houtryve <pierre.vanhoutryve@amd.com> | 2024-02-01 08:53:32 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-02-01 08:53:32 +0100 |
commit | 7ec996d4c5c30083b070be4898140440094e6b97 (patch) | |
tree | adfd86a95b3cc13418a2b00d5195f5a4ec5b5e3f | |
parent | 65066c02770cc3da3b5154fbb7ed9df78ab94b93 (diff) |
[GlobalISel][TableGen] Support Intrinsics in MIR Patterns (#79278)
14 files changed, 313 insertions, 22 deletions
diff --git a/llvm/docs/GlobalISel/MIRPatterns.rst b/llvm/docs/GlobalISel/MIRPatterns.rst index cbbe962dcb81..728e32470144 100644 --- a/llvm/docs/GlobalISel/MIRPatterns.rst +++ b/llvm/docs/GlobalISel/MIRPatterns.rst @@ -36,8 +36,8 @@ MIR patterns use the DAG datatype in TableGen. (inst operand0, operand1, ...) -``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``) -or ``GICombinePatFrag``. +``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``), +``Intrinsic`` or ``GICombinePatFrag``. Operands essentially fall into one of two categories: @@ -227,7 +227,6 @@ Limitations This a non-exhaustive list of known issues with MIR patterns at this time. -* Matching intrinsics is not yet possible. * Using ``GICombinePatFrag`` within another ``GICombinePatFrag`` is not supported. * ``GICombinePatFrag`` can only have a single root. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h index d399eb6de2b2..29a46f04fd5d 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h @@ -379,6 +379,11 @@ enum { /// - Flags(2) - Register Flags GIR_AddRegister, + /// Adds an intrinsic ID to the specified instruction. + /// - InsnID(ULEB128) - Instruction ID to modify + /// - IID(2) - Intrinsic ID + GIR_AddIntrinsicID, + /// Marks the implicit def of a register as dead. /// - InsnID(ULEB128) - Instruction ID to modify /// - OpIdx(ULEB128) - The implicit def operand index diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h index 0a2709ef216b..8e456015cd37 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h @@ -1116,6 +1116,16 @@ bool GIMatchTableExecutor::executeMatchTable( << "], " << RegNum << ", " << RegFlags << ")\n"); break; } + case GIR_AddIntrinsicID: { + uint64_t InsnID = readULEB(); + uint16_t Value = readU16(); + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value); + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs[" + << InsnID << "], " << Value << ")\n"); + break; + } case GIR_SetImplicitDefDead: { uint64_t InsnID = readULEB(); uint64_t OpIdx = readULEB(); diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td new file mode 100644 index 000000000000..90d04f752384 --- /dev/null +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td @@ -0,0 +1,10 @@ +// Dummy intrinsic definitions for TableGen. + + +def int_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; +def int_0in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [], []>; + +def int_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrHasSideEffects]>; + +def int_convergent_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>; +def int_convergent_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrHasSideEffects]>; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td index f9aa926591e4..8e3e27daaa5b 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td @@ -53,7 +53,7 @@ def TestPF: GICombinePatFrag< (outs root:$def), (ins), [(pattern (COPY $def, $src))]>; -// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction or Intrinsic def eraseroot_notinstmatch: GICombineRule< (defs root:$mi), (match (TestPF $dst):$mi), diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td new file mode 100644 index 000000000000..94cc3e58dfc9 --- /dev/null +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td @@ -0,0 +1,86 @@ +// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \ +// RUN: -combiners=MyCombiner %s | \ +// RUN: FileCheck %s + +include "llvm/Target/Target.td" +include "llvm/Target/GlobalISel/Combine.td" + +include "test-intrinsics.td" + +def MyTargetISA : InstrInfo; +def MyTarget : Target { let InstructionSet = MyTargetISA; } + +def IntrinTest0 : GICombineRule< + (defs root:$a), + (match (int_1in_1out $a, 0)), + (apply (int_1in_1out $a, $x), + (int_0in_1out i32:$x))>; + +def SpecialIntrins : GICombineRule< + (defs root:$a), + (match (int_sideeffects_1in_1out $a, $b)), + (apply (int_convergent_1in_1out i32:$x, $b), + (int_convergent_sideeffects_1in_1out $a, $x))>; + +def MyCombiner: GICombiner<"GenMyCombiner", [ + IntrinTest0, + SpecialIntrins +]>; + + +// CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { +// CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(114), GIMT_Encode2(116), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132), +// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18), +// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73), +// CHECK-NEXT: // Label 0: @18 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(72), // Rule ID 0 // +// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, +// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out), +// CHECK-NEXT: // MIs[0] a +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // MIs[0] Operand 2 +// CHECK-NEXT: GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 0, +// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32, +// CHECK-NEXT: // Combiner Rule #0: IntrinTest0 +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC), +// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define), +// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::0in_1out), +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a +// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::1in_1out), +// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0, +// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, +// CHECK-NEXT: GIR_Done, +// CHECK-NEXT: // Label 3: @72 +// CHECK-NEXT: GIM_Reject, +// CHECK-NEXT: // Label 1: @73 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(131), // Rule ID 1 // +// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, +// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out), +// CHECK-NEXT: // MIs[0] a +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // MIs[0] b +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32, +// CHECK-NEXT: // Combiner Rule #1: SpecialIntrins +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT), +// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define), +// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::convergent_1in_1out), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // b +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a +// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::convergent_sideeffects_1in_1out), +// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0, +// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0, +// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, +// CHECK-NEXT: GIR_Done, +// CHECK-NEXT: // Label 4: @131 +// CHECK-NEXT: GIM_Reject, +// CHECK-NEXT: // Label 2: @132 +// CHECK-NEXT: GIM_Reject, +// CHECK-NEXT: }; // Size: 133 bytes +// CHECK-NEXT: return MatchTable0; +// CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td index 318438b977dc..4b214d2ca89e 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td @@ -1,10 +1,12 @@ -// RUN: not llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \ +// RUN: not llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \ // RUN: -combiners=MyCombiner %s 2>&1| \ // RUN: FileCheck %s -implicit-check-not=error: include "llvm/Target/Target.td" include "llvm/Target/GlobalISel/Combine.td" +include "test-intrinsics.td" + def MyTargetISA : InstrInfo; def MyTarget : Target { let InstructionSet = MyTargetISA; } @@ -248,6 +250,13 @@ def miflags_in_builtin : GICombineRule< (match (COPY $x, $y)), (apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>; +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: matching/writing MIFlags is only allowed on CodeGenInstruction patterns +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$x, ?:$y, (MIFlags FmArcp))' +def miflags_in_intrin : GICombineRule< + (defs root:$x), + (match (int_1in_1out $x, $y)), + (apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>; + // CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'match' patterns cannot refer to flags from other instructions // CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: MIFlags in 'mi' refer to: impostor def using_flagref_in_match : GICombineRule< @@ -300,6 +309,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ not_miflagenum_1, not_miflagenum_2, miflags_in_builtin, + miflags_in_intrin, using_flagref_in_match, badflagref_in_apply ]>; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td index 26f3bb88da95..c261ec457545 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td @@ -1,10 +1,12 @@ -// RUN: llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \ +// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \ // RUN: -gicombiner-stop-after-parse -combiners=MyCombiner %s | \ // RUN: FileCheck %s include "llvm/Target/Target.td" include "llvm/Target/GlobalISel/Combine.td" +include "test-intrinsics.td" + def MyTargetISA : InstrInfo; def MyTarget : Target { let InstructionSet = MyTargetISA; } @@ -342,6 +344,48 @@ def MIFlagsTest : GICombineRule< (match (G_ZEXT $dst, $src, (MIFlags FmReassoc, (not FmNoNans, FmArcp))):$mi), (apply (G_MUL $dst, $src, $src, (MIFlags $mi, FmReassoc, (not FmNsz, FmArcp))))>; +// CHECK-NEXT: (CombineRule name:IntrinTest0 id:12 root:a +// CHECK-NEXT: (MatchPats +// CHECK-NEXT: <match_root>__IntrinTest0_match_0:(CodeGenInstructionPattern G_INTRINSIC operands:[<def>$a, $b] intrinsic(@llvm.1in.1out)) +// CHECK-NEXT: ) +// CHECK-NEXT: (ApplyPats +// CHECK-NEXT: <apply_root>__IntrinTest0_apply_0:(CodeGenInstructionPattern G_INTRINSIC_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.sideeffects.1in.1out)) +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable MatchPats +// CHECK-NEXT: a -> __IntrinTest0_match_0 +// CHECK-NEXT: b -> <live-in> +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable ApplyPats +// CHECK-NEXT: a -> __IntrinTest0_apply_0 +// CHECK-NEXT: b -> <live-in> +// CHECK-NEXT: ) +// CHECK-NEXT: ) +def IntrinTest0 : GICombineRule< + (defs root:$a), + (match (int_1in_1out $a, $b)), + (apply (int_sideeffects_1in_1out $a, $b))>; + +// CHECK: (CombineRule name:IntrinTest1 id:13 root:a +// CHECK-NEXT: (MatchPats +// CHECK-NEXT: <match_root>__IntrinTest1_match_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT operands:[<def>$a, $b] intrinsic(@llvm.convergent.1in.1out)) +// CHECK-NEXT: ) +// CHECK-NEXT: (ApplyPats +// CHECK-NEXT: <apply_root>__IntrinTest1_apply_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.convergent.sideeffects.1in.1out)) +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable MatchPats +// CHECK-NEXT: a -> __IntrinTest1_match_0 +// CHECK-NEXT: b -> <live-in> +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable ApplyPats +// CHECK-NEXT: a -> __IntrinTest1_apply_0 +// CHECK-NEXT: b -> <live-in> +// CHECK-NEXT: ) +// CHECK-NEXT: ) +def IntrinTest1 : GICombineRule< + (defs root:$a), + (match (int_convergent_1in_1out $a, $b)), + (apply (int_convergent_sideeffects_1in_1out $a, $b))>; + def MyCombiner: GICombiner<"GenMyCombiner", [ WipOpcodeTest0, WipOpcodeTest1, @@ -354,5 +398,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ VariadicsInTest, VariadicsOutTest, TypeOfTest, - MIFlagsTest + MIFlagsTest, + IntrinTest0, + IntrinTest1 ]>; diff --git a/llvm/test/TableGen/lit.local.cfg b/llvm/test/TableGen/lit.local.cfg index 6d3ea5578453..0e827479cd41 100644 --- a/llvm/test/TableGen/lit.local.cfg +++ b/llvm/test/TableGen/lit.local.cfg @@ -1,2 +1,2 @@ config.suffixes = [".td"] -config.excludes = ["Common"] +config.excludes = ["Common", "Inputs"] diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/GlobalISel/Patterns.cpp index 0a6d05e06dca..758eac2dfebd 100644 --- a/llvm/utils/TableGen/GlobalISel/Patterns.cpp +++ b/llvm/utils/TableGen/GlobalISel/Patterns.cpp @@ -8,6 +8,7 @@ #include "Patterns.h" #include "../CodeGenInstruction.h" +#include "../CodeGenIntrinsics.h" #include "CXXPredicates.h" #include "CodeExpander.h" #include "CodeExpansions.h" @@ -331,7 +332,7 @@ bool CodeGenInstructionPattern::is(StringRef OpcodeName) const { } bool CodeGenInstructionPattern::isVariadic() const { - return I.Operands.isVariadic; + return !isIntrinsic() && I.Operands.isVariadic; } bool CodeGenInstructionPattern::hasVariadicDefs() const { @@ -352,6 +353,9 @@ bool CodeGenInstructionPattern::hasVariadicDefs() const { } unsigned CodeGenInstructionPattern::getNumInstDefs() const { + if (isIntrinsic()) + return IntrinInfo->IS.RetTys.size(); + if (!isVariadic() || !hasVariadicDefs()) return I.Operands.NumDefs; unsigned NumOuts = I.Operands.size() - I.Operands.NumDefs; @@ -360,6 +364,9 @@ unsigned CodeGenInstructionPattern::getNumInstDefs() const { } unsigned CodeGenInstructionPattern::getNumInstOperands() const { + if (isIntrinsic()) + return IntrinInfo->IS.RetTys.size() + IntrinInfo->IS.ParamTys.size(); + unsigned NumCGIOps = I.Operands.size(); return isVariadic() ? std::max<unsigned>(NumCGIOps, Operands.size()) : NumCGIOps; @@ -376,6 +383,9 @@ StringRef CodeGenInstructionPattern::getInstName() const { } void CodeGenInstructionPattern::printExtras(raw_ostream &OS) const { + if (isIntrinsic()) + OS << " intrinsic(@" << IntrinInfo->Name << ")"; + if (!FI) return; diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/GlobalISel/Patterns.h index b3160552a21f..dac092556548 100644 --- a/llvm/utils/TableGen/GlobalISel/Patterns.h +++ b/llvm/utils/TableGen/GlobalISel/Patterns.h @@ -34,6 +34,7 @@ class SMLoc; class StringInit; class CodeExpansions; class CodeGenInstruction; +struct CodeGenIntrinsic; namespace gi { @@ -396,7 +397,7 @@ private: StringMap<InstructionPattern *> Table; }; -//===- CodeGenInstructionPattern ------------------------------------------===// +//===- MIFlagsInfo --------------------------------------------------------===// /// Helper class to contain data associated with a MIFlags operand. class MIFlagsInfo { @@ -413,7 +414,17 @@ private: SetVector<StringRef> SetF, UnsetF, CopyF; }; -/// Matches an instruction, e.g. `G_ADD $x, $y, $z`. +//===- CodeGenInstructionPattern ------------------------------------------===// + +/// Matches an instruction or intrinsic: +/// e.g. `G_ADD $x, $y, $z` or `int_amdgcn_cos $a` +/// +/// Intrinsics are just normal instructions with a special operand for intrinsic +/// ID. Despite G_INTRINSIC opcodes being variadic, we consider that the +/// Intrinsic's info takes priority. This means we return: +/// - false for isVariadic() and other variadic-related queries. +/// - getNumInstDefs and getNumInstOperands use the intrinsic's in/out +/// operands. class CodeGenInstructionPattern : public InstructionPattern { public: CodeGenInstructionPattern(const CodeGenInstruction &I, StringRef Name) @@ -425,6 +436,10 @@ public: bool is(StringRef OpcodeName) const; + void setIntrinsic(const CodeGenIntrinsic *I) { IntrinInfo = I; } + const CodeGenIntrinsic *getIntrinsic() const { return IntrinInfo; } + bool isIntrinsic() const { return IntrinInfo; } + bool hasVariadicDefs() const; bool isVariadic() const override; unsigned getNumInstDefs() const override; @@ -440,6 +455,7 @@ private: void printExtras(raw_ostream &OS) const override; const CodeGenInstruction &I; + const CodeGenIntrinsic *IntrinInfo = nullptr; std::unique_ptr<MIFlagsInfo> FI; }; diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp index 124e416eea28..d9249cf627f2 100644 --- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -27,6 +27,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenInstruction.h" +#include "CodeGenIntrinsics.h" #include "CodeGenTarget.h" #include "GlobalISel/CXXPredicates.h" #include "GlobalISel/CodeExpander.h" @@ -401,9 +402,9 @@ void CombineRuleOperandTypeChecker::propagateAndInferTypes() { PatternType CombineRuleOperandTypeChecker::inferImmediateType( const InstructionPattern &IP, unsigned ImmOpIdx, const TypeEquivalenceClasses &TECs) const { - // We can only infer CGPs. + // We can only infer CGPs (except intrinsics). const auto *CGP = dyn_cast<CodeGenInstructionPattern>(&IP); - if (!CGP) + if (!CGP || CGP->isIntrinsic()) return {}; // For CGPs, we try to infer immediates by trying to infer another named @@ -521,14 +522,18 @@ void CombineRuleOperandTypeChecker::getInstEqClasses( // - Iterating over the map, filtering types we don't like, and just adding // the array of Operand Indexes to \p OutTECs. - // We can only do this on CodeGenInstructions. Other InstructionPatterns have - // no type inference information associated with them. + // We can only do this on CodeGenInstructions that aren't intrinsics. Other + // InstructionPatterns have no type inference information associated with + // them. + // TODO: We could try to extract some info from CodeGenIntrinsic to + // guide inference. + // TODO: Could we add some inference information to builtins at least? e.g. // ReplaceReg should always replace with a reg of the same type, for instance. // Though, those patterns are often used alone so it might not be worth the // trouble to infer their types. auto *CGP = dyn_cast<CodeGenInstructionPattern>(&P); - if (!CGP) + if (!CGP || CGP->isIntrinsic()) return; const auto MCOITypes = getMCOIOperandTypes(*CGP); @@ -1251,8 +1256,8 @@ bool CombineRuleBuilder::checkSemantics() { const auto *IRoot = dyn_cast<CodeGenInstructionPattern>(MatchRoot); if (!IRoot) { - PrintError(Name + - " can only be used if the root is a CodeGenInstruction"); + PrintError(Name + " can only be used if the root is a " + "CodeGenInstruction or Intrinsic"); return false; } @@ -1546,6 +1551,32 @@ bool CombineRuleBuilder::parsePatternList( return true; } +static const CodeGenInstruction & +getInstrForIntrinsic(const CodeGenTarget &CGT, const CodeGenIntrinsic *I) { + StringRef Opc; + if (I->isConvergent) { + Opc = I->hasSideEffects ? "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS" + : "G_INTRINSIC_CONVERGENT"; + } else { + Opc = I->hasSideEffects ? "G_INTRINSIC_W_SIDE_EFFECTS" : "G_INTRINSIC"; + } + + RecordKeeper &RK = I->TheDef->getRecords(); + return CGT.getInstruction(RK.getDef(Opc)); +} + +static const CodeGenIntrinsic *getCodeGenIntrinsic(Record *R) { + // Intrinsics need to have a static lifetime because the match table keeps + // references to CodeGenIntrinsic objects. + static DenseMap<const Record *, std::unique_ptr<CodeGenIntrinsic>> + AllIntrinsics; + + auto &Ptr = AllIntrinsics[R]; + if (!Ptr) + Ptr = std::make_unique<CodeGenIntrinsic>(R, std::vector<Record *>()); + return Ptr.get(); +} + std::unique_ptr<Pattern> CombineRuleBuilder::parseInstructionPattern(const Init &Arg, StringRef Name) const { @@ -1558,6 +1589,14 @@ CombineRuleBuilder::parseInstructionPattern(const Init &Arg, auto &Instr = CGT.getInstruction(IP->getOperatorAsDef(RuleDef.getLoc())); Pat = std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name)); + } else if (const DagInit *IP = + getDagWithOperatorOfSubClass(Arg, "Intrinsic")) { + Record *TheDef = IP->getOperatorAsDef(RuleDef.getLoc()); + const CodeGenIntrinsic *Intrin = getCodeGenIntrinsic(TheDef); + const CodeGenInstruction &Instr = getInstrForIntrinsic(CGT, Intrin); + Pat = + std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name)); + cast<CodeGenInstructionPattern>(*Pat).setIntrinsic(Intrin); } else if (const DagInit *PFP = getDagWithOperatorOfSubClass(Arg, PatFrag::ClassName)) { const Record *Def = PFP->getOperatorAsDef(RuleDef.getLoc()); @@ -1569,9 +1608,8 @@ CombineRuleBuilder::parseInstructionPattern(const Init &Arg, getDagWithOperatorOfSubClass(Arg, BuiltinPattern::ClassName)) { Pat = std::make_unique<BuiltinPattern>( *BP->getOperatorAsDef(RuleDef.getLoc()), insertStrRef(Name)); - } else { + } else return nullptr; - } for (unsigned K = 0; K < DagPat->getNumArgs(); ++K) { Init *Arg = DagPat->getArg(K); @@ -2237,7 +2275,18 @@ bool CombineRuleBuilder::emitInstructionApplyPattern( auto &DstMI = M.addAction<BuildMIAction>(M.allocateOutputInsnID(), &CGIP.getInst()); + bool HasEmittedIntrinsicID = false; + const auto EmitIntrinsicID = [&]() { + assert(CGIP.isIntrinsic()); + DstMI.addRenderer<IntrinsicIDRenderer>(CGIP.getIntrinsic()); + HasEmittedIntrinsicID = true; + }; + for (auto &Op : P.operands()) { + // Emit the intrinsic ID after the last def. + if (CGIP.isIntrinsic() && !Op.isDef() && !HasEmittedIntrinsicID) + EmitIntrinsicID(); + if (Op.isNamedImmediate()) { PrintError("invalid output operand '" + Op.getOperandName() + "': output immediates cannot be named"); @@ -2326,6 +2375,11 @@ bool CombineRuleBuilder::emitInstructionApplyPattern( DstMI.addRenderer<TempRegRenderer>(TempRegID, /*IsDef=*/true); } + // Some intrinsics have no in operands, ensure the ID is still emitted in such + // cases. + if (CGIP.isIntrinsic() && !HasEmittedIntrinsicID) + EmitIntrinsicID(); + // Render MIFlags if (const auto *FI = CGIP.getMIFlagsInfo()) { for (StringRef InstName : FI->copy_flags()) @@ -2455,6 +2509,14 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern( IM.addPredicate<InstructionOpcodeMatcher>(&P.getInst()); declareInstExpansion(CE, IM, P.getName()); + // If this is an intrinsic, check the intrinsic ID. + if (P.isIntrinsic()) { + // The IntrinsicID's operand is the first operand after the defs. + OperandMatcher &OM = IM.addOperand(P.getNumInstDefs(), "$intrinsic_id", + AllocatedTemporariesBaseID++); + OM.addPredicate<IntrinsicIDOperandMatcher>(P.getIntrinsic()); + } + // Check flags if needed. if (const auto *FI = P.getMIFlagsInfo()) { assert(FI->copy_flags().empty()); @@ -2466,7 +2528,7 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern( /*CheckNot=*/true); } - for (const auto &[Idx, OriginalO] : enumerate(P.operands())) { + for (auto [Idx, OriginalO] : enumerate(P.operands())) { // Remap the operand. This is used when emitting InstructionPatterns inside // PatFrags, so it can remap them to the arguments passed to the pattern. // @@ -2481,8 +2543,17 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern( const auto OpName = RemappedO.isNamedOperand() ? RemappedO.getOperandName().str() : ""; + + // For intrinsics, the first use operand is the intrinsic id, so the true + // operand index is shifted by 1. + // + // From now on: + // Idx = index in the pattern operand list. + // RealIdx = expected index in the MachineInstr. + const unsigned RealIdx = + (P.isIntrinsic() && !OriginalO.isDef()) ? (Idx + 1) : Idx; OperandMatcher &OM = - IM.addOperand(Idx, OpName, AllocatedTemporariesBaseID++); + IM.addOperand(RealIdx, OpName, AllocatedTemporariesBaseID++); if (!OpName.empty()) declareOperandExpansion(CE, OM, OriginalO.getOperandName()); diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp index 1ae8e30da2fb..051c536f113f 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp @@ -2030,6 +2030,16 @@ void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table, Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; } +//===- IntrinsicIDRenderer ------------------------------------------------===// + +void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table, + RuleMatcher &Rule) const { + Table << MatchTable::Opcode("GIR_AddIntrinsicID") << MatchTable::Comment("MI") + << MatchTable::ULEB128Value(InsnID) + << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName) + << MatchTable::LineBreak; +} + //===- CustomRenderer -----------------------------------------------------===// void CustomRenderer::emitRenderOpcodes(MatchTable &Table, diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h index 9c7bd9e33028..635552fc8904 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/GlobalISelMatchTable.h @@ -1845,6 +1845,7 @@ public: OR_Register, OR_TempRegister, OR_ComplexPattern, + OR_Intrinsic, OR_Custom, OR_CustomOperand }; @@ -2135,6 +2136,23 @@ public: void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; }; +/// Adds an intrinsic ID operand to the instruction being built. +class IntrinsicIDRenderer : public OperandRenderer { +protected: + unsigned InsnID; + const CodeGenIntrinsic *II; + +public: + IntrinsicIDRenderer(unsigned InsnID, const CodeGenIntrinsic *II) + : OperandRenderer(OR_Intrinsic), InsnID(InsnID), II(II) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_Intrinsic; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; +}; + class CustomRenderer : public OperandRenderer { protected: unsigned InsnID; |