[GlobalISel][TableGen] Support Intrinsics in MIR Patterns (#79278)

14 files changed, 313 insertions, 22 deletions

diff --git a/llvm/docs/GlobalISel/MIRPatterns.rst b/llvm/docs/GlobalISel/MIRPatterns.rst
index cbbe962dcb81..728e32470144 100644
--- a/llvm/docs/GlobalISel/MIRPatterns.rst
+++ b/llvm/docs/GlobalISel/MIRPatterns.rst
@@ -36,8 +36,8 @@ MIR patterns use the DAG datatype in TableGen.
 
   (inst operand0, operand1, ...)
 
-``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``)
-or ``GICombinePatFrag``.
+``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``),
+``Intrinsic`` or ``GICombinePatFrag``.
 
 Operands essentially fall into one of two categories:
 
@@ -227,7 +227,6 @@ Limitations
 
 This a non-exhaustive list of known issues with MIR patterns at this time.
 
-* Matching intrinsics is not yet possible.
 * Using ``GICombinePatFrag`` within another ``GICombinePatFrag`` is not
   supported.
 * ``GICombinePatFrag`` can only have a single root.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index d399eb6de2b2..29a46f04fd5d 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -379,6 +379,11 @@ enum {
   /// - Flags(2) - Register Flags
   GIR_AddRegister,
 
+  /// Adds an intrinsic ID to the specified instruction.
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - IID(2) - Intrinsic ID
+  GIR_AddIntrinsicID,
+
   /// Marks the implicit def of a register as dead.
   /// - InsnID(ULEB128) - Instruction ID to modify
   /// - OpIdx(ULEB128) - The implicit def operand index
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 0a2709ef216b..8e456015cd37 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -1116,6 +1116,16 @@ bool GIMatchTableExecutor::executeMatchTable(
                           << "], " << RegNum << ", " << RegFlags << ")\n");
       break;
     }
+    case GIR_AddIntrinsicID: {
+      uint64_t InsnID = readULEB();
+      uint16_t Value = readU16();
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
+      DEBUG_WITH_TYPE(TgtExecutor::getName(),
+                      dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
+                             << InsnID << "], " << Value << ")\n");
+      break;
+    }
     case GIR_SetImplicitDefDead: {
       uint64_t InsnID = readULEB();
       uint64_t OpIdx = readULEB();
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td
new file mode 100644
index 000000000000..90d04f752384
--- /dev/null
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td
@@ -0,0 +1,10 @@
+// Dummy intrinsic definitions for TableGen.
+
+
+def int_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], []>;
+def int_0in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [], []>;
+
+def int_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrHasSideEffects]>;
+
+def int_convergent_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>;
+def int_convergent_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrHasSideEffects]>;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td
index f9aa926591e4..8e3e27daaa5b 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td
@@ -53,7 +53,7 @@ def TestPF: GICombinePatFrag<
     (outs root:$def),
     (ins),
     [(pattern (COPY $def, $src))]>;
-// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction or Intrinsic
 def eraseroot_notinstmatch: GICombineRule<
   (defs root:$mi),
   (match (TestPF $dst):$mi),
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td
new file mode 100644
index 000000000000..94cc3e58dfc9
--- /dev/null
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td
@@ -0,0 +1,86 @@
+// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
+// RUN:     -combiners=MyCombiner %s | \
+// RUN: FileCheck %s
+
+include "llvm/Target/Target.td"
+include "llvm/Target/GlobalISel/Combine.td"
+
+include "test-intrinsics.td"
+
+def MyTargetISA : InstrInfo;
+def MyTarget : Target { let InstructionSet = MyTargetISA; }
+
+def IntrinTest0 : GICombineRule<
+  (defs root:$a),
+  (match (int_1in_1out $a, 0)),
+  (apply (int_1in_1out $a, $x),
+         (int_0in_1out i32:$x))>;
+
+def SpecialIntrins : GICombineRule<
+  (defs root:$a),
+  (match (int_sideeffects_1in_1out $a, $b)),
+  (apply (int_convergent_1in_1out i32:$x, $b),
+         (int_convergent_sideeffects_1in_1out $a, $x))>;
+
+def MyCombiner: GICombiner<"GenMyCombiner", [
+  IntrinTest0,
+  SpecialIntrins
+]>;
+
+
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(114), GIMT_Encode2(116), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132),
+// CHECK-NEXT:     /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18),
+// CHECK-NEXT:     /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73),
+// CHECK-NEXT:     // Label 0: @18
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(72), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:       GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out),
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] Operand 2
+// CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 0,
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
+// CHECK-NEXT:       // Combiner Rule #0: IntrinTest0
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
+// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT:       GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::0in_1out),
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
+// CHECK-NEXT:       GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::1in_1out),
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 3: @72
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     // Label 1: @73
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(131), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:       GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out),
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] b
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
+// CHECK-NEXT:       // Combiner Rule #1: SpecialIntrins
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT),
+// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT:       GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::convergent_1in_1out),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // b
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
+// CHECK-NEXT:       GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::convergent_sideeffects_1in_1out),
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
+// CHECK-NEXT:       GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 4: @131
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     // Label 2: @132
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     }; // Size: 133 bytes
+// CHECK-NEXT:   return MatchTable0;
+// CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
index 318438b977dc..4b214d2ca89e 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
@@ -1,10 +1,12 @@
-// RUN: not llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
+// RUN: not llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
 // RUN:     -combiners=MyCombiner %s 2>&1| \
 // RUN: FileCheck %s -implicit-check-not=error:
 
 include "llvm/Target/Target.td"
 include "llvm/Target/GlobalISel/Combine.td"
 
+include "test-intrinsics.td"
+
 def MyTargetISA : InstrInfo;
 def MyTarget : Target { let InstructionSet = MyTargetISA; }
 
@@ -248,6 +250,13 @@ def miflags_in_builtin : GICombineRule<
   (match (COPY $x, $y)),
   (apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>;
 
+// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: matching/writing MIFlags is only allowed on CodeGenInstruction patterns
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$x, ?:$y, (MIFlags FmArcp))'
+def miflags_in_intrin : GICombineRule<
+  (defs root:$x),
+  (match (int_1in_1out $x, $y)),
+  (apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>;
+
 // CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'match' patterns cannot refer to flags from other instructions
 // CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: MIFlags in 'mi' refer to: impostor
 def using_flagref_in_match : GICombineRule<
@@ -300,6 +309,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   not_miflagenum_1,
   not_miflagenum_2,
   miflags_in_builtin,
+  miflags_in_intrin,
   using_flagref_in_match,
   badflagref_in_apply
 ]>;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
index 26f3bb88da95..c261ec457545 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
@@ -1,10 +1,12 @@
-// RUN: llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
+// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
 // RUN:     -gicombiner-stop-after-parse -combiners=MyCombiner %s | \
 // RUN: FileCheck %s
 
 include "llvm/Target/Target.td"
 include "llvm/Target/GlobalISel/Combine.td"
 
+include "test-intrinsics.td"
+
 def MyTargetISA : InstrInfo;
 def MyTarget : Target { let InstructionSet = MyTargetISA; }
 
@@ -342,6 +344,48 @@ def MIFlagsTest : GICombineRule<
   (match (G_ZEXT $dst, $src, (MIFlags FmReassoc, (not FmNoNans, FmArcp))):$mi),
   (apply (G_MUL $dst, $src, $src, (MIFlags $mi, FmReassoc, (not FmNsz, FmArcp))))>;
 
+// CHECK-NEXT: (CombineRule name:IntrinTest0 id:12 root:a
+// CHECK-NEXT:   (MatchPats
+// CHECK-NEXT:     <match_root>__IntrinTest0_match_0:(CodeGenInstructionPattern G_INTRINSIC operands:[<def>$a, $b] intrinsic(@llvm.1in.1out))
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (ApplyPats
+// CHECK-NEXT:     <apply_root>__IntrinTest0_apply_0:(CodeGenInstructionPattern G_INTRINSIC_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.sideeffects.1in.1out))
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable MatchPats
+// CHECK-NEXT:     a -> __IntrinTest0_match_0
+// CHECK-NEXT:     b -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable ApplyPats
+// CHECK-NEXT:     a -> __IntrinTest0_apply_0
+// CHECK-NEXT:     b -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT: )
+def IntrinTest0 : GICombineRule<
+  (defs root:$a),
+  (match (int_1in_1out $a, $b)),
+  (apply (int_sideeffects_1in_1out $a, $b))>;
+
+// CHECK:      (CombineRule name:IntrinTest1 id:13 root:a
+// CHECK-NEXT:   (MatchPats
+// CHECK-NEXT:     <match_root>__IntrinTest1_match_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT operands:[<def>$a, $b] intrinsic(@llvm.convergent.1in.1out))
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (ApplyPats
+// CHECK-NEXT:     <apply_root>__IntrinTest1_apply_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.convergent.sideeffects.1in.1out))
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable MatchPats
+// CHECK-NEXT:     a -> __IntrinTest1_match_0
+// CHECK-NEXT:     b -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable ApplyPats
+// CHECK-NEXT:     a -> __IntrinTest1_apply_0
+// CHECK-NEXT:     b -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT: )
+def IntrinTest1 : GICombineRule<
+  (defs root:$a),
+  (match (int_convergent_1in_1out $a, $b)),
+  (apply (int_convergent_sideeffects_1in_1out $a, $b))>;
+
 def MyCombiner: GICombiner<"GenMyCombiner", [
   WipOpcodeTest0,
   WipOpcodeTest1,
@@ -354,5 +398,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   VariadicsInTest,
   VariadicsOutTest,
   TypeOfTest,
-  MIFlagsTest
+  MIFlagsTest,
+  IntrinTest0,
+  IntrinTest1
 ]>;
diff --git a/llvm/test/TableGen/lit.local.cfg b/llvm/test/TableGen/lit.local.cfg
index 6d3ea5578453..0e827479cd41 100644
--- a/llvm/test/TableGen/lit.local.cfg
+++ b/llvm/test/TableGen/lit.local.cfg
@@ -1,2 +1,2 @@
 config.suffixes = [".td"]
-config.excludes = ["Common"]
+config.excludes = ["Common", "Inputs"]
diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/GlobalISel/Patterns.cpp
index 0a6d05e06dca..758eac2dfebd 100644
--- a/llvm/utils/TableGen/GlobalISel/Patterns.cpp
+++ b/llvm/utils/TableGen/GlobalISel/Patterns.cpp
@@ -8,6 +8,7 @@
 
 #include "Patterns.h"
 #include "../CodeGenInstruction.h"
+#include "../CodeGenIntrinsics.h"
 #include "CXXPredicates.h"
 #include "CodeExpander.h"
 #include "CodeExpansions.h"
@@ -331,7 +332,7 @@ bool CodeGenInstructionPattern::is(StringRef OpcodeName) const {
 }
 
 bool CodeGenInstructionPattern::isVariadic() const {
-  return I.Operands.isVariadic;
+  return !isIntrinsic() && I.Operands.isVariadic;
 }
 
 bool CodeGenInstructionPattern::hasVariadicDefs() const {
@@ -352,6 +353,9 @@ bool CodeGenInstructionPattern::hasVariadicDefs() const {
 }
 
 unsigned CodeGenInstructionPattern::getNumInstDefs() const {
+  if (isIntrinsic())
+    return IntrinInfo->IS.RetTys.size();
+
   if (!isVariadic() || !hasVariadicDefs())
     return I.Operands.NumDefs;
   unsigned NumOuts = I.Operands.size() - I.Operands.NumDefs;
@@ -360,6 +364,9 @@ unsigned CodeGenInstructionPattern::getNumInstDefs() const {
 }
 
 unsigned CodeGenInstructionPattern::getNumInstOperands() const {
+  if (isIntrinsic())
+    return IntrinInfo->IS.RetTys.size() + IntrinInfo->IS.ParamTys.size();
+
   unsigned NumCGIOps = I.Operands.size();
   return isVariadic() ? std::max<unsigned>(NumCGIOps, Operands.size())
                       : NumCGIOps;
@@ -376,6 +383,9 @@ StringRef CodeGenInstructionPattern::getInstName() const {
 }
 
 void CodeGenInstructionPattern::printExtras(raw_ostream &OS) const {
+  if (isIntrinsic())
+    OS << " intrinsic(@" << IntrinInfo->Name << ")";
+
   if (!FI)
     return;
 
diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/GlobalISel/Patterns.h
index b3160552a21f..dac092556548 100644
--- a/llvm/utils/TableGen/GlobalISel/Patterns.h
+++ b/llvm/utils/TableGen/GlobalISel/Patterns.h
@@ -34,6 +34,7 @@ class SMLoc;
 class StringInit;
 class CodeExpansions;
 class CodeGenInstruction;
+struct CodeGenIntrinsic;
 
 namespace gi {
 
@@ -396,7 +397,7 @@ private:
   StringMap<InstructionPattern *> Table;
 };
 
-//===- CodeGenInstructionPattern ------------------------------------------===//
+//===- MIFlagsInfo --------------------------------------------------------===//
 
 /// Helper class to contain data associated with a MIFlags operand.
 class MIFlagsInfo {
@@ -413,7 +414,17 @@ private:
   SetVector<StringRef> SetF, UnsetF, CopyF;
 };
 
-/// Matches an instruction, e.g. `G_ADD $x, $y, $z`.
+//===- CodeGenInstructionPattern ------------------------------------------===//
+
+/// Matches an instruction or intrinsic:
+///    e.g. `G_ADD $x, $y, $z` or `int_amdgcn_cos $a`
+///
+/// Intrinsics are just normal instructions with a special operand for intrinsic
+/// ID. Despite G_INTRINSIC opcodes being variadic, we consider that the
+/// Intrinsic's info takes priority. This means we return:
+///   - false for isVariadic() and other variadic-related queries.
+///   - getNumInstDefs and getNumInstOperands use the intrinsic's in/out
+///   operands.
 class CodeGenInstructionPattern : public InstructionPattern {
 public:
   CodeGenInstructionPattern(const CodeGenInstruction &I, StringRef Name)
@@ -425,6 +436,10 @@ public:
 
   bool is(StringRef OpcodeName) const;
 
+  void setIntrinsic(const CodeGenIntrinsic *I) { IntrinInfo = I; }
+  const CodeGenIntrinsic *getIntrinsic() const { return IntrinInfo; }
+  bool isIntrinsic() const { return IntrinInfo; }
+
   bool hasVariadicDefs() const;
   bool isVariadic() const override;
   unsigned getNumInstDefs() const override;
@@ -440,6 +455,7 @@ private:
   void printExtras(raw_ostream &OS) const override;
 
   const CodeGenInstruction &I;
+  const CodeGenIntrinsic *IntrinInfo = nullptr;
   std::unique_ptr<MIFlagsInfo> FI;
 };
 
diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index 124e416eea28..d9249cf627f2 100644
--- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -27,6 +27,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenInstruction.h"
+#include "CodeGenIntrinsics.h"
 #include "CodeGenTarget.h"
 #include "GlobalISel/CXXPredicates.h"
 #include "GlobalISel/CodeExpander.h"
@@ -401,9 +402,9 @@ void CombineRuleOperandTypeChecker::propagateAndInferTypes() {
 PatternType CombineRuleOperandTypeChecker::inferImmediateType(
     const InstructionPattern &IP, unsigned ImmOpIdx,
     const TypeEquivalenceClasses &TECs) const {
-  // We can only infer CGPs.
+  // We can only infer CGPs (except intrinsics).
   const auto *CGP = dyn_cast<CodeGenInstructionPattern>(&IP);
-  if (!CGP)
+  if (!CGP || CGP->isIntrinsic())
     return {};
 
   // For CGPs, we try to infer immediates by trying to infer another named
@@ -521,14 +522,18 @@ void CombineRuleOperandTypeChecker::getInstEqClasses(
   //    - Iterating over the map, filtering types we don't like, and just adding
   //      the array of Operand Indexes to \p OutTECs.
 
-  // We can only do this on CodeGenInstructions. Other InstructionPatterns have
-  // no type inference information associated with them.
+  // We can only do this on CodeGenInstructions that aren't intrinsics. Other
+  // InstructionPatterns have no type inference information associated with
+  // them.
+  // TODO: We could try to extract some info from CodeGenIntrinsic to
+  //       guide inference.
+
   // TODO: Could we add some inference information to builtins at least? e.g.
   // ReplaceReg should always replace with a reg of the same type, for instance.
   // Though, those patterns are often used alone so it might not be worth the
   // trouble to infer their types.
   auto *CGP = dyn_cast<CodeGenInstructionPattern>(&P);
-  if (!CGP)
+  if (!CGP || CGP->isIntrinsic())
     return;
 
   const auto MCOITypes = getMCOIOperandTypes(*CGP);
@@ -1251,8 +1256,8 @@ bool CombineRuleBuilder::checkSemantics() {
 
       const auto *IRoot = dyn_cast<CodeGenInstructionPattern>(MatchRoot);
       if (!IRoot) {
-        PrintError(Name +
-                   " can only be used if the root is a CodeGenInstruction");
+        PrintError(Name + " can only be used if the root is a "
+                          "CodeGenInstruction or Intrinsic");
         return false;
       }
 
@@ -1546,6 +1551,32 @@ bool CombineRuleBuilder::parsePatternList(
   return true;
 }
 
+static const CodeGenInstruction &
+getInstrForIntrinsic(const CodeGenTarget &CGT, const CodeGenIntrinsic *I) {
+  StringRef Opc;
+  if (I->isConvergent) {
+    Opc = I->hasSideEffects ? "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS"
+                            : "G_INTRINSIC_CONVERGENT";
+  } else {
+    Opc = I->hasSideEffects ? "G_INTRINSIC_W_SIDE_EFFECTS" : "G_INTRINSIC";
+  }
+
+  RecordKeeper &RK = I->TheDef->getRecords();
+  return CGT.getInstruction(RK.getDef(Opc));
+}
+
+static const CodeGenIntrinsic *getCodeGenIntrinsic(Record *R) {
+  // Intrinsics need to have a static lifetime because the match table keeps
+  // references to CodeGenIntrinsic objects.
+  static DenseMap<const Record *, std::unique_ptr<CodeGenIntrinsic>>
+      AllIntrinsics;
+
+  auto &Ptr = AllIntrinsics[R];
+  if (!Ptr)
+    Ptr = std::make_unique<CodeGenIntrinsic>(R, std::vector<Record *>());
+  return Ptr.get();
+}
+
 std::unique_ptr<Pattern>
 CombineRuleBuilder::parseInstructionPattern(const Init &Arg,
                                             StringRef Name) const {
@@ -1558,6 +1589,14 @@ CombineRuleBuilder::parseInstructionPattern(const Init &Arg,
     auto &Instr = CGT.getInstruction(IP->getOperatorAsDef(RuleDef.getLoc()));
     Pat =
         std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name));
+  } else if (const DagInit *IP =
+                 getDagWithOperatorOfSubClass(Arg, "Intrinsic")) {
+    Record *TheDef = IP->getOperatorAsDef(RuleDef.getLoc());
+    const CodeGenIntrinsic *Intrin = getCodeGenIntrinsic(TheDef);
+    const CodeGenInstruction &Instr = getInstrForIntrinsic(CGT, Intrin);
+    Pat =
+        std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name));
+    cast<CodeGenInstructionPattern>(*Pat).setIntrinsic(Intrin);
   } else if (const DagInit *PFP =
                  getDagWithOperatorOfSubClass(Arg, PatFrag::ClassName)) {
     const Record *Def = PFP->getOperatorAsDef(RuleDef.getLoc());
@@ -1569,9 +1608,8 @@ CombineRuleBuilder::parseInstructionPattern(const Init &Arg,
                  getDagWithOperatorOfSubClass(Arg, BuiltinPattern::ClassName)) {
     Pat = std::make_unique<BuiltinPattern>(
         *BP->getOperatorAsDef(RuleDef.getLoc()), insertStrRef(Name));
-  } else {
+  } else
     return nullptr;
-  }
 
   for (unsigned K = 0; K < DagPat->getNumArgs(); ++K) {
     Init *Arg = DagPat->getArg(K);
@@ -2237,7 +2275,18 @@ bool CombineRuleBuilder::emitInstructionApplyPattern(
   auto &DstMI =
       M.addAction<BuildMIAction>(M.allocateOutputInsnID(), &CGIP.getInst());
 
+  bool HasEmittedIntrinsicID = false;
+  const auto EmitIntrinsicID = [&]() {
+    assert(CGIP.isIntrinsic());
+    DstMI.addRenderer<IntrinsicIDRenderer>(CGIP.getIntrinsic());
+    HasEmittedIntrinsicID = true;
+  };
+
   for (auto &Op : P.operands()) {
+    // Emit the intrinsic ID after the last def.
+    if (CGIP.isIntrinsic() && !Op.isDef() && !HasEmittedIntrinsicID)
+      EmitIntrinsicID();
+
     if (Op.isNamedImmediate()) {
       PrintError("invalid output operand '" + Op.getOperandName() +
                  "': output immediates cannot be named");
@@ -2326,6 +2375,11 @@ bool CombineRuleBuilder::emitInstructionApplyPattern(
     DstMI.addRenderer<TempRegRenderer>(TempRegID, /*IsDef=*/true);
   }
 
+  // Some intrinsics have no in operands, ensure the ID is still emitted in such
+  // cases.
+  if (CGIP.isIntrinsic() && !HasEmittedIntrinsicID)
+    EmitIntrinsicID();
+
   // Render MIFlags
   if (const auto *FI = CGIP.getMIFlagsInfo()) {
     for (StringRef InstName : FI->copy_flags())
@@ -2455,6 +2509,14 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern(
   IM.addPredicate<InstructionOpcodeMatcher>(&P.getInst());
   declareInstExpansion(CE, IM, P.getName());
 
+  // If this is an intrinsic, check the intrinsic ID.
+  if (P.isIntrinsic()) {
+    // The IntrinsicID's operand is the first operand after the defs.
+    OperandMatcher &OM = IM.addOperand(P.getNumInstDefs(), "$intrinsic_id",
+                                       AllocatedTemporariesBaseID++);
+    OM.addPredicate<IntrinsicIDOperandMatcher>(P.getIntrinsic());
+  }
+
   // Check flags if needed.
   if (const auto *FI = P.getMIFlagsInfo()) {
     assert(FI->copy_flags().empty());
@@ -2466,7 +2528,7 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern(
                                                           /*CheckNot=*/true);
   }
 
-  for (const auto &[Idx, OriginalO] : enumerate(P.operands())) {
+  for (auto [Idx, OriginalO] : enumerate(P.operands())) {
     // Remap the operand. This is used when emitting InstructionPatterns inside
     // PatFrags, so it can remap them to the arguments passed to the pattern.
     //
@@ -2481,8 +2543,17 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern(
 
     const auto OpName =
         RemappedO.isNamedOperand() ? RemappedO.getOperandName().str() : "";
+
+    // For intrinsics, the first use operand is the intrinsic id, so the true
+    // operand index is shifted by 1.
+    //
+    // From now on:
+    //    Idx = index in the pattern operand list.
+    //    RealIdx = expected index in the MachineInstr.
+    const unsigned RealIdx =
+        (P.isIntrinsic() && !OriginalO.isDef()) ? (Idx + 1) : Idx;
     OperandMatcher &OM =
-        IM.addOperand(Idx, OpName, AllocatedTemporariesBaseID++);
+        IM.addOperand(RealIdx, OpName, AllocatedTemporariesBaseID++);
     if (!OpName.empty())
       declareOperandExpansion(CE, OM, OriginalO.getOperandName());
 
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
index 1ae8e30da2fb..051c536f113f 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
@@ -2030,6 +2030,16 @@ void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table,
   Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
+//===- IntrinsicIDRenderer ------------------------------------------------===//
+
+void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table,
+                                            RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIR_AddIntrinsicID") << MatchTable::Comment("MI")
+        << MatchTable::ULEB128Value(InsnID)
+        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName)
+        << MatchTable::LineBreak;
+}
+
 //===- CustomRenderer -----------------------------------------------------===//
 
 void CustomRenderer::emitRenderOpcodes(MatchTable &Table,
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h
index 9c7bd9e33028..635552fc8904 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.h
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.h
@@ -1845,6 +1845,7 @@ public:
     OR_Register,
     OR_TempRegister,
     OR_ComplexPattern,
+    OR_Intrinsic,
     OR_Custom,
     OR_CustomOperand
   };
@@ -2135,6 +2136,23 @@ public:
   void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
 };
 
+/// Adds an intrinsic ID operand to the instruction being built.
+class IntrinsicIDRenderer : public OperandRenderer {
+protected:
+  unsigned InsnID;
+  const CodeGenIntrinsic *II;
+
+public:
+  IntrinsicIDRenderer(unsigned InsnID, const CodeGenIntrinsic *II)
+      : OperandRenderer(OR_Intrinsic), InsnID(InsnID), II(II) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_Intrinsic;
+  }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
 class CustomRenderer : public OperandRenderer {
 protected:
   unsigned InsnID;