[llvm] [GlobalISel] Add G_PREFETCH (PR #74863)

Jay Foad via llvm-commits llvm-commits at lists.llvm.org
Fri Dec 8 08:48:13 PST 2023


https://github.com/jayfoad created https://github.com/llvm/llvm-project/pull/74863

None

>From e406c734609d3cd1ae436084c42c1c63d8af2795 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Fri, 8 Dec 2023 14:08:09 +0000
Subject: [PATCH] [GlobalISel] Add G_PREFETCH

---
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  4 ++
 llvm/include/llvm/Support/TargetOpcodes.def   |  3 +
 llvm/include/llvm/Target/GenericOpcodes.td    |  9 +++
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  | 12 ++++
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   | 10 +++
 llvm/lib/CodeGen/MachineVerifier.cpp          | 23 +++++++
 llvm/lib/IR/Verifier.cpp                      |  2 +-
 llvm/lib/Target/AArch64/AArch64InstrGISel.td  |  4 +-
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    | 55 ++++++++--------
 .../AArch64/GISel/AArch64LegalizerInfo.h      |  1 +
 .../GlobalISel/legalizer-info-validation.mir  |  3 +
 llvm/test/MachineVerifier/test_g_prefetch.mir | 40 ++++++++++++
 .../builtins/match-table-replacerreg.td       | 20 +++---
 .../match-table-imms.td                       | 28 ++++-----
 .../match-table-patfrag-root.td               |  2 +-
 .../GlobalISelCombinerEmitter/match-table.td  | 62 +++++++++----------
 llvm/test/TableGen/GlobalISelEmitter.td       |  2 +-
 17 files changed, 195 insertions(+), 85 deletions(-)
 create mode 100644 llvm/test/MachineVerifier/test_g_prefetch.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 3d36d06a7e9da..eb846acde3e04 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1529,6 +1529,10 @@ class MachineIRBuilder {
   /// Build and insert `G_FENCE Ordering, Scope`.
   MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope);
 
+  /// Build and insert G_PREFETCH \p Addr, \p RW, \p Locality, \p CacheType
+  MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW,
+                                    unsigned Locality, unsigned CacheType);
+
   /// Build and insert \p Dst = G_FREEZE \p Src
   MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src) {
     return buildInstr(TargetOpcode::G_FREEZE, {Dst}, {Src});
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 941c6d5f8cad8..91d9eb745a48f 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -415,6 +415,9 @@ HANDLE_TARGET_OPCODE_MARKER(GENERIC_ATOMICRMW_OP_END, G_ATOMICRMW_UDEC_WRAP)
 // Generic atomic fence
 HANDLE_TARGET_OPCODE(G_FENCE)
 
+/// Generic prefetch
+HANDLE_TARGET_OPCODE(G_PREFETCH)
+
 /// Generic conditional branch instruction.
 HANDLE_TARGET_OPCODE(G_BRCOND)
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 9a9c09d3c20d6..73e38b15bf671 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1209,6 +1209,15 @@ def G_FENCE : GenericInstruction {
   let hasSideEffects = true;
 }
 
+// Generic opcode equivalent to the llvm.prefetch intrinsic.
+def G_PREFETCH : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins ptype0:$address, i32imm:$rw, i32imm:$locality, i32imm:$cachetype);
+  let hasSideEffects = true;
+  let mayLoad = true;
+  let mayStore = true;
+}
+
 //------------------------------------------------------------------------------
 // Variadic ops
 //------------------------------------------------------------------------------
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 14a4e72152e7c..b2850846bde67 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2435,6 +2435,18 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {});
     return true;
   }
+  case Intrinsic::prefetch: {
+    Value *Addr = CI.getOperand(0);
+    ConstantInt *RW = cast<ConstantInt>(CI.getOperand(1));
+    ConstantInt *Locality = cast<ConstantInt>(CI.getOperand(2));
+    ConstantInt *CacheType = cast<ConstantInt>(CI.getOperand(3));
+
+    MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW->getZExtValue(),
+                             Locality->getZExtValue(),
+                             CacheType->getZExtValue());
+
+    return true;
+  }
 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)  \
   case Intrinsic::INTRINSIC:
 #include "llvm/IR/ConstrainedOps.def"
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 80e9c08e850b6..f7febc9357c11 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -1051,6 +1051,16 @@ MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
     .addImm(Scope);
 }
 
+MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr,
+                                                    unsigned RW,
+                                                    unsigned Locality,
+                                                    unsigned CacheType) {
+  auto MIB = buildInstr(TargetOpcode::G_PREFETCH);
+  Addr.addSrcToMIB(MIB);
+  MIB.addImm(RW).addImm(Locality).addImm(CacheType);
+  return MIB;
+}
+
 MachineInstrBuilder
 MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
 #ifndef NDEBUG
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index aaf9bd740d137..a015d9bbd2d3f 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1812,6 +1812,29 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
     }
     break;
   }
+  case TargetOpcode::G_PREFETCH: {
+    const MachineOperand &AddrOp = MI->getOperand(0);
+    if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer()) {
+      report("addr operand must be a pointer", &AddrOp, 0);
+      break;
+    }
+    const MachineOperand &RWOp = MI->getOperand(1);
+    if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) {
+      report("rw operand must be an immediate 0-1", &RWOp, 1);
+      break;
+    }
+    const MachineOperand &LocalityOp = MI->getOperand(2);
+    if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) {
+      report("locality operand must be an immediate 0-3", &LocalityOp, 2);
+      break;
+    }
+    const MachineOperand &CacheTypeOp = MI->getOperand(3);
+    if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) {
+      report("cache type operand must be an immediate 0-1", &CacheTypeOp, 3);
+      break;
+    }
+    break;
+  }
   case TargetOpcode::G_ASSERT_ALIGN: {
     if (MI->getOperand(2).getImm() < 1)
       report("alignment immediate must be >= 1", MI);
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index f137f0468c3c5..c87f164bdd0f5 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5374,7 +5374,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
     Check(cast<ConstantInt>(Call.getArgOperand(1))->getZExtValue() < 2,
           "rw argument to llvm.prefetch must be 0-1", Call);
     Check(cast<ConstantInt>(Call.getArgOperand(2))->getZExtValue() < 4,
-          "locality argument to llvm.prefetch must be 0-4", Call);
+          "locality argument to llvm.prefetch must be 0-3", Call);
     Check(cast<ConstantInt>(Call.getArgOperand(3))->getZExtValue() < 2,
           "cache type argument to llvm.prefetch must be 0-1", Call);
     break;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrGISel.td b/llvm/lib/Target/AArch64/AArch64InstrGISel.td
index 1711360779bf7..1c88456560d3d 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrGISel.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrGISel.td
@@ -209,7 +209,7 @@ def G_FCMLTZ : AArch64GenericInstruction {
   let hasSideEffects = 0;
 }
 
-def G_PREFETCH : AArch64GenericInstruction {
+def G_AARCH64_PREFETCH : AArch64GenericInstruction {
   let OutOperandList = (outs);
   let InOperandList = (ins type0:$imm, ptype0:$src1);
   let hasSideEffects = 1;
@@ -287,7 +287,7 @@ def : GINodeEquiv<G_SDOT, AArch64sdot>;
 
 def : GINodeEquiv<G_EXTRACT_VECTOR_ELT, vector_extract>;
 
-def : GINodeEquiv<G_PREFETCH, AArch64Prefetch>;
+def : GINodeEquiv<G_AARCH64_PREFETCH, AArch64Prefetch>;
 
 // These are patterns that we only use for GlobalISel via the importer.
 def : Pat<(f32 (fadd (vector_extract (v2f32 FPR64:$Rn), (i64 0)),
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 21a412e9360dc..a35957c34a596 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1127,6 +1127,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   getActionDefinitionsBuilder(G_IS_FPCLASS).lower();
 
+  getActionDefinitionsBuilder(G_PREFETCH).custom();
+
   getLegacyLegalizerInfo().computeTables();
   verify(*ST.getInstrInfo());
 }
@@ -1176,6 +1178,8 @@ bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
     return legalizeExtractVectorElt(MI, MRI, Helper);
   case TargetOpcode::G_DYN_STACKALLOC:
     return legalizeDynStackAlloc(MI, Helper);
+  case TargetOpcode::G_PREFETCH:
+    return legalizePrefetch(MI, Helper);
   }
 
   llvm_unreachable("expected switch to return");
@@ -1349,30 +1353,6 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
     Value.setReg(ExtValueReg);
     return true;
   }
-  case Intrinsic::prefetch: {
-    MachineIRBuilder MIB(MI);
-    auto &AddrVal = MI.getOperand(1);
-
-    int64_t IsWrite = MI.getOperand(2).getImm();
-    int64_t Locality = MI.getOperand(3).getImm();
-    int64_t IsData = MI.getOperand(4).getImm();
-
-    bool IsStream = Locality == 0;
-    if (Locality != 0) {
-      assert(Locality <= 3 && "Prefetch locality out-of-range");
-      // The locality degree is the opposite of the cache speed.
-      // Put the number the other way around.
-      // The encoding starts at 0 for level 1
-      Locality = 3 - Locality;
-    }
-
-    unsigned PrfOp =
-        (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
-
-    MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal);
-    MI.eraseFromParent();
-    return true;
-  }
   case Intrinsic::aarch64_prefetch: {
     MachineIRBuilder MIB(MI);
     auto &AddrVal = MI.getOperand(1);
@@ -1387,7 +1367,7 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
                      (Target << 1) |     // Cache level bits
                      (unsigned)IsStream; // Stream bit
 
-    MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal);
+    MIB.buildInstr(AArch64::G_AARCH64_PREFETCH).addImm(PrfOp).add(AddrVal);
     MI.eraseFromParent();
     return true;
   }
@@ -1986,3 +1966,28 @@ bool AArch64LegalizerInfo::legalizeDynStackAlloc(
   MI.eraseFromParent();
   return true;
 }
+
+bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
+                                            LegalizerHelper &Helper) const {
+  MachineIRBuilder &MIB = Helper.MIRBuilder;
+  auto &AddrVal = MI.getOperand(0);
+
+  int64_t IsWrite = MI.getOperand(1).getImm();
+  int64_t Locality = MI.getOperand(2).getImm();
+  int64_t IsData = MI.getOperand(3).getImm();
+
+  bool IsStream = Locality == 0;
+  if (Locality != 0) {
+    assert(Locality <= 3 && "Prefetch locality out-of-range");
+    // The locality degree is the opposite of the cache speed.
+    // Put the number the other way around.
+    // The encoding starts at 0 for level 1
+    Locality = 3 - Locality;
+  }
+
+  unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
+
+  MIB.buildInstr(AArch64::G_AARCH64_PREFETCH).addImm(PrfOp).add(AddrVal);
+  MI.eraseFromParent();
+  return true;
+}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 6fd859d334cd8..19f77baa77f89 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -64,6 +64,7 @@ class AArch64LegalizerInfo : public LegalizerInfo {
   bool legalizeExtractVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI,
                                 LegalizerHelper &Helper) const;
   bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const;
+  bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const;
   const AArch64Subtarget *ST;
 };
 } // End llvm namespace.
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index ae15e74a43277..178db852e35b7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -258,6 +258,9 @@
 # DEBUG-NEXT: G_FENCE (opcode {{[0-9]+}}): 0 type indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_PREFETCH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_BRCOND (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
diff --git a/llvm/test/MachineVerifier/test_g_prefetch.mir b/llvm/test/MachineVerifier/test_g_prefetch.mir
new file mode 100644
index 0000000000000..a08b0803fc355
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_g_prefetch.mir
@@ -0,0 +1,40 @@
+# RUN: not --crash llc -o - -mtriple=aarch64 -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+
+---
+name: test_fcmp
+legalized: true
+regBankSelected: false
+selected: false
+tracksRegLiveness: true
+liveins:
+body: |
+  bb.0:
+    liveins: $x0, $w0, $q0
+    %s32:_(s32) = COPY $w0
+    %ptr:_(p0) = COPY $x0
+
+    G_PREFETCH %ptr
+    ; CHECK: *** Bad machine code: Too few operands ***
+    ; CHECK: 4 operands expected, but 1 given.
+
+    G_PREFETCH %ptr, 0, 0, 0, 0
+    ; CHECK: *** Bad machine code: Extra explicit operand on non-variadic instruction ***
+    ; CHECK: operand 4:
+
+    G_PREFETCH %s32, 0, 0, 0
+    ; CHECK: *** Bad machine code: addr operand must be a pointer ***
+    ; CHECK: operand 0:
+
+    G_PREFETCH %ptr, 10, 0, 0
+    ; CHECK: *** Bad machine code: rw operand must be an immediate 0-1 ***
+    ; CHECK: operand 1:
+
+    G_PREFETCH %ptr, 0, 10, 0
+    ; CHECK: *** Bad machine code: locality operand must be an immediate 0-3 ***
+    ; CHECK: operand 2:
+
+    G_PREFETCH %ptr, 0, 0, 10
+    ; CHECK: *** Bad machine code: cache type operand must be an immediate 0-1 ***
+    ; CHECK: operand 3:
+...
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td
index 2d968977701fd..6ae1305aa1aa5 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td
@@ -28,11 +28,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/65, 180, /*)*//*default:*//*Label 2*/ 192,
-// CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 1*/ 165,
-// CHECK-NEXT:     // Label 0: @120
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ 164, // Rule ID 1 //
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/65, 181, /*)*//*default:*//*Label 2*/ 193,
+// CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 1*/ 166,
+// CHECK-NEXT:     // Label 0: @121
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ 165, // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
 // CHECK-NEXT:       // MIs[0] a
@@ -57,10 +57,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_ReplaceRegWithTempReg, /*OldInsnID*/0, /*OldOpIdx*/1, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 3: @164
+// CHECK-NEXT:     // Label 3: @165
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @165
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 191, // Rule ID 0 //
+// CHECK-NEXT:     // Label 1: @166
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 192, // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
@@ -75,9 +75,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_ReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 4: @191
+// CHECK-NEXT:     // Label 4: @192
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @192
+// CHECK-NEXT:     // Label 2: @193
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
index 0495a66a7c577..fd5f7db0b4f12 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
@@ -34,12 +34,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 3*/ 194,
-// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_CONSTANT*//*Label 1*/ 138, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 2*/ 165,
-// CHECK-NEXT:     // Label 0: @112
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 137, // Rule ID 0 //
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 127, /*)*//*default:*//*Label 3*/ 195,
+// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_CONSTANT*//*Label 1*/ 139, 0, 0, 0, 0, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 2*/ 166,
+// CHECK-NEXT:     // Label 0: @113
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 138, // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[0] a
@@ -51,10 +51,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_AddImm, /*InsnID*/0, /*Imm*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 4: @137
+// CHECK-NEXT:     // Label 4: @138
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @138
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ 164, // Rule ID 2 //
+// CHECK-NEXT:     // Label 1: @139
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ 165, // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled,
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[0] a
@@ -66,10 +66,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/42,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 5: @164
+// CHECK-NEXT:     // Label 5: @165
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @165
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 193, // Rule ID 1 //
+// CHECK-NEXT:     // Label 2: @166
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 194, // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -83,9 +83,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 6: @193
+// CHECK-NEXT:     // Label 6: @194
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @194
+// CHECK-NEXT:     // Label 3: @195
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
index 5cb9206ca5f2c..b62ebcf246874 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
@@ -28,7 +28,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/118, 181, /*)*//*default:*//*Label 3*/ 152,
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/119, 182, /*)*//*default:*//*Label 3*/ 152,
 // CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 0*/ 68, 0, 0, 0, 0, 0, 0,
 // CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 1*/ 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 // CHECK-NEXT:     /*TargetOpcode::G_FPEXT*//*Label 2*/ 127,
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
index 3b0f656924322..6777089f846f2 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
@@ -132,15 +132,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // Verify match table.
 // CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 6*/ 267,
-// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 3*/ 216, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 4*/ 231, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 5*/ 239,
-// CHECK-NEXT:     // Label 0: @112
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ 133, // Rule ID 4 //
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 127, /*)*//*default:*//*Label 6*/ 268,
+// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 3*/ 217, 0, 0, 0, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 4*/ 232, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 5*/ 240,
+// CHECK-NEXT:     // Label 0: @113
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ 134, // Rule ID 4 //
 // CHECK-NEXT:       GIM_CheckFeatures, GIFBS_HasAnswerToEverything,
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule3Enabled,
 // CHECK-NEXT:       // MIs[0] a
@@ -155,8 +155,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // Combiner Rule #3: InstTest1
 // CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 7: @133
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ 140, // Rule ID 3 //
+// CHECK-NEXT:     // Label 7: @134
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ 141, // Rule ID 3 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // Combiner Rule #2: InstTest0
 // CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner1,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 8: @140
+// CHECK-NEXT:     // Label 8: @141
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @141
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 9*/ 180, // Rule ID 6 //
+// CHECK-NEXT:     // Label 1: @142
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 9*/ 181, // Rule ID 6 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule5Enabled,
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[0] dst
@@ -186,10 +186,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 9: @180
+// CHECK-NEXT:     // Label 9: @181
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @181
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 10*/ 215, // Rule ID 5 //
+// CHECK-NEXT:     // Label 2: @182
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 10*/ 216, // Rule ID 5 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule4Enabled,
 // CHECK-NEXT:       // MIs[0] tmp
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1]
@@ -207,32 +207,32 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner2,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 10: @215
+// CHECK-NEXT:     // Label 10: @216
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @216
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 11*/ 223, // Rule ID 0 //
+// CHECK-NEXT:     // Label 3: @217
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 11*/ 224, // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:       // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC'
 // CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 11: @223
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 12*/ 230, // Rule ID 1 //
+// CHECK-NEXT:     // Label 11: @224
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 12*/ 231, // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
 // CHECK-NEXT:       // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC'
 // CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 12: @230
+// CHECK-NEXT:     // Label 12: @231
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 4: @231
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 13*/ 238, // Rule ID 2 //
+// CHECK-NEXT:     // Label 4: @232
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 13*/ 239, // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
 // CHECK-NEXT:       // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT'
 // CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 13: @238
+// CHECK-NEXT:     // Label 13: @239
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 5: @239
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ 266, // Rule ID 7 //
+// CHECK-NEXT:     // Label 5: @240
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ 267, // Rule ID 7 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule6Enabled,
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
@@ -247,10 +247,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 14: @266
+// CHECK-NEXT:     // Label 14: @267
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 6: @267
+// CHECK-NEXT:     // Label 6: @268
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     }; // Size: 2144 bytes
+// CHECK-NEXT:     }; // Size: 2152 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 176db59a1f412..01f807b4fb6b3 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -517,7 +517,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
 // R00O-NEXT:  GIM_Reject,
 // R00O:       // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
 // R00O-NEXT:  GIM_Reject,
-// R00O-NEXT:  }; // Size: 9784 bytes
+// R00O-NEXT:  }; // Size: 9792 bytes
 
 def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
                  [(set GPR32:$dst,



More information about the llvm-commits mailing list