[llvm] [AArch64][SPIRV][GlobalISel] Migrate target-specific wip_match_opcode combines to MIR-pattern (PR #222903)

Vikash Gupta via llvm-commits llvm-commits at lists.llvm.org
Mon Sep 28 03:36:12 PDT 2026


https://github.com/vg0204 updated https://github.com/llvm/llvm-project/pull/222903

>From 6cdaadc102aa2f5d22c15aa0ce9945614b116e40 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Fri, 11 Sep 2026 15:16:11 +0530
Subject: [PATCH 1/2] [AArch64][SPIRV][GlobalISel] Migrate wip_match_opcode
 combines to MIR-pattern.

It replaces the `wip_match_opcode` with declarative MIR-pattern
match roots across the AArch64 and SPIRV target-specific GICombines.

Some important points to consider :

- **shuffle_vector_lowering:** these `G_SHUFFLE_VECTOR` rules are order-
  sensitive. `fullrev` previously gained priority through a nested
  `G_IMPLICIT_DEF` match; that predicate is relocated to C++ so all
  rules are equal-priority and `fullrev` is ordered last, preserving
  the original dispatch sequence and codegen.
- **SPIRV intrinsic roots:** the matrix/length/distance rules match and
  apply on target intrinsics (`int_spv_*`, `int_matrix_*`), requiring the
  `IntrinsicsSPIRV` enum in the combiner translation unit.
- **Deferred:** `vector_unmerge_lowering` and `unmerge_ext_to_unmerge`
  remain on `wip_match_opcode`; their `G_UNMERGE_VALUES` variadic-def
  roots are not yet expressible as MIR patterns.
---
 llvm/lib/Target/AArch64/AArch64Combine.td     |  77 ++-
 .../GISel/AArch64PostLegalizerCombiner.cpp    |  63 --
 llvm/lib/Target/SPIRV/SPIRVCombine.td         |  19 +-
 llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp |  33 -
 llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h   |   2 -
 .../SPIRV/SPIRVPreLegalizerCombiner.cpp       |   1 +
 .../postlegalizer-combiner-anyext-to-zext.mir |   2 +-
 .../AArch64/arm64-extract-insert-varidx.ll    |  10 +-
 llvm/test/CodeGen/AArch64/insertextract.ll    |  16 +-
 .../AArch64/neon-shuffle-vector-tbl.ll        | 631 +++++++-----------
 10 files changed, 296 insertions(+), 558 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index 7a8083e3ceb44..af3675264e7d5 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -21,7 +21,7 @@ def icmp_redundant_trunc : GICombineRule<
 def fold_global_offset_matchdata : GIDefMatchData<"std::pair<uint64_t, uint64_t>">;
 def fold_global_offset : GICombineRule<
   (defs root:$root, fold_global_offset_matchdata:$matchinfo),
-  (match (wip_match_opcode G_GLOBAL_VALUE):$root,
+  (match (G_GLOBAL_VALUE $dst, $gv):$root,
           [{ return matchFoldGlobalOffset(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyFoldGlobalOffset(*${root}, MRI, B, Observer, ${matchinfo});}])
 >;
@@ -31,7 +31,7 @@ def ext_addv_to_udot_addv_matchinfo : GIDefMatchData<"std::tuple<Register, Regis
 let Predicates = [HasDotProd] in {
 def ext_addv_to_udot_addv : GICombineRule<
   (defs root:$root, ext_addv_to_udot_addv_matchinfo:$matchinfo),
-  (match (wip_match_opcode G_VECREDUCE_ADD):$root,
+  (match (G_VECREDUCE_ADD $dst, $src):$root,
          [{ return matchExtAddvToDotAddv(*${root}, MRI, STI, ${matchinfo}); }]),
   (apply [{ applyExtAddvToDotAddv(*${root}, MRI, B, Observer, STI, ${matchinfo}); }])
 >;
@@ -40,7 +40,7 @@ def ext_addv_to_udot_addv : GICombineRule<
 def ext_uaddv_to_uaddlv_matchinfo : GIDefMatchData<"std::pair<Register, bool>">;
 def ext_uaddv_to_uaddlv : GICombineRule<
   (defs root:$root, ext_uaddv_to_uaddlv_matchinfo:$matchinfo),
-  (match (wip_match_opcode G_VECREDUCE_ADD):$root,
+  (match (G_VECREDUCE_ADD $dst, $src):$root,
          [{ return matchExtUaddvToUaddlv(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyExtUaddvToUaddlv(*${root}, MRI, B, Observer, ${matchinfo}); }])
 >;
@@ -110,58 +110,60 @@ def shuffle_matchdata : GIDefMatchData<"ShuffleVectorPseudo">;
 
 def rev : GICombineRule<
   (defs root:$root, shuffle_matchdata:$matchinfo),
-  (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+  (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root,
          [{ return matchREV(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyShuffleVectorPseudo(*${root}, MRI, ${matchinfo}); }])
 >;
 
 def zip : GICombineRule<
   (defs root:$root, shuffle_matchdata:$matchinfo),
-  (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+  (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root,
          [{ return matchZip(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyShuffleVectorPseudo(*${root}, MRI, ${matchinfo}); }])
 >;
 
 def uzp : GICombineRule<
   (defs root:$root, shuffle_matchdata:$matchinfo),
-  (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+  (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root,
          [{ return matchUZP(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyShuffleVectorPseudo(*${root}, MRI, ${matchinfo}); }])
 >;
 
 def dup: GICombineRule <
   (defs root:$root, shuffle_matchdata:$matchinfo),
-  (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+  (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root,
          [{ return matchDup(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyShuffleVectorPseudo(*${root}, MRI, ${matchinfo}); }])
 >;
 
 def trn : GICombineRule<
   (defs root:$root, shuffle_matchdata:$matchinfo),
-  (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+  (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root,
          [{ return matchTRN(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyShuffleVectorPseudo(*${root}, MRI, ${matchinfo}); }])
 >;
 
 def ext: GICombineRule <
   (defs root:$root, shuffle_matchdata:$matchinfo),
-  (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+  (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root,
          [{ return matchEXT(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyEXT(*${root}, ${matchinfo}); }])
 >;
 
+// src2 == G_IMPLICIT_DEF is checked in C++, not as a nested operand match, to
+// keep fullrev equal-priority with the other shuffle rules (so it stays last).
 def fullrev: GICombineRule <
   (defs root:$root, shuffle_matchdata:$matchinfo),
-  (match (G_IMPLICIT_DEF $src2),
-         (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask):$root,
-         [{ return ShuffleVectorInst::isReverseMask(${mask}.getShuffleMask(),
+  (match (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask):$root,
+         [{ return mi_match(${src2}.getReg(), MRI, m_GImplicitDef()) &&
+                   ShuffleVectorInst::isReverseMask(${mask}.getShuffleMask(),
                                                     ${mask}.getShuffleMask().size()); }]),
   (apply [{ applyFullRev(*${root}, MRI); }])
 >;
 
 def insertelt_nonconst: GICombineRule <
   (defs root:$root, shuffle_matchdata:$matchinfo),
-  (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
+  (match (G_INSERT_VECTOR_ELT $dst, $src, $elt, $idx):$root,
          [{ return matchNonConstInsert(*${root}, MRI); }]),
   (apply [{ applyNonConstInsert(*${root}, MRI, B); }])
 >;
@@ -169,23 +171,26 @@ def insertelt_nonconst: GICombineRule <
 def shuf_to_ins_matchdata : GIDefMatchData<"std::tuple<Register, int, Register, int>">;
 def shuf_to_ins: GICombineRule <
   (defs root:$root, shuf_to_ins_matchdata:$matchinfo),
-  (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+  (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root,
          [{ return matchINS(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyINS(*${root}, MRI, B, ${matchinfo}); }])
 >;
 
+def vashr_vlshr_imm_frags : GICombinePatFrag<
+  (outs root:$dst), (ins),
+  !foreach(op, [G_ASHR, G_LSHR], (pattern (op $dst, $src, $amt)))>;
 def vashr_vlshr_imm : GICombineRule<
   (defs root:$root, int64_matchinfo:$matchinfo),
-  (match (wip_match_opcode G_ASHR, G_LSHR):$root,
-          [{ return matchVAshrLshrImm(*${root}, MRI, ${matchinfo}); }]),
-  (apply [{ applyVAshrLshrImm(*${root}, MRI, ${matchinfo}); }])
+  (match (vashr_vlshr_imm_frags $root):$mi,
+          [{ return matchVAshrLshrImm(*${mi}, MRI, ${matchinfo}); }]),
+  (apply [{ applyVAshrLshrImm(*${mi}, MRI, ${matchinfo}); }])
 >;
 
 def form_duplane_matchdata :
   GIDefMatchData<"std::pair<unsigned, int>">;
 def form_duplane : GICombineRule <
   (defs root:$root, form_duplane_matchdata:$matchinfo),
-  (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+  (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root,
           [{ return matchDupLane(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyDupLane(*${root}, MRI, B, ${matchinfo}); }])
 >;
@@ -220,7 +225,7 @@ def unmerge_duplanes : GICombineGroup<[unmerge_duplane8, unmerge_duplane16,
                                        extract_duplane16, extract_duplane32]>;
 
 def shuffle_vector_lowering : GICombineGroup<[dup, form_duplane, rev, ext, zip,
-                                              uzp, trn, fullrev, shuf_to_ins]>;
+                                              uzp, trn, shuf_to_ins, fullrev]>;
 
 // Turn G_UNMERGE_VALUES -> G_EXTRACT_VECTOR_ELT's
 def vector_unmerge_lowering : GICombineRule <
@@ -251,7 +256,7 @@ def icmp_lowering : GICombineGroup<[adjust_icmp_imm, swap_icmp_operands]>;
 def extractvecelt_pairwise_add_matchdata : GIDefMatchData<"std::tuple<unsigned, LLT, Register>">;
 def extractvecelt_pairwise_add : GICombineRule<
   (defs root:$root, extractvecelt_pairwise_add_matchdata:$matchinfo),
-  (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
+  (match (G_EXTRACT_VECTOR_ELT $dst, $src, $idx):$root,
           [{ return matchExtractVecEltPairwiseAdd(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyExtractVecEltPairwiseAdd(*${root}, MRI, B, ${matchinfo}); }])
 >;
@@ -259,7 +264,7 @@ def extractvecelt_pairwise_add : GICombineRule<
 def mul_const_matchdata : GIDefMatchData<"std::function<void(MachineIRBuilder&, Register)>">;
 def mul_const : GICombineRule<
   (defs root:$root, mul_const_matchdata:$matchinfo),
-  (match (wip_match_opcode G_MUL):$root,
+  (match (G_MUL $dst, $src1, $src2):$root,
           [{ return matchAArch64MulConstCombine(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyAArch64MulConstCombine(*${root}, MRI, B, ${matchinfo}); }])
 >;
@@ -274,13 +279,14 @@ def extmultomull : GICombineRule<
 
 // The mid end will reassociate sub(sub(x, m1), m2) to sub(x, add(m1, m2))
 // This reassociates it back to allow the creation of more mls instructions.
+// src3 - (src1 + src2) -> (src3 - src1) - src2, when src1/src2 are muls.
 def subaddmulreassoc : GICombineRule<
   (defs root:$dst),
   (match (G_ADD $tmp, $src1, $src2):$sub,
          (G_SUB $dst, $src3, $tmp):$root,
          [{ return matchSubAddMulReassoc(${src1}.getReg(), ${src2}.getReg(),
                                          ${tmp}.getReg(), ${src3}.getReg(), MRI); }]),
-  (apply [{ applySubAddMulReassoc(*${root}, *${sub}, MRI, B, Observer); }])>;
+  (apply (G_SUB $t, $src3, $src1), (G_SUB $dst, $t, $src2))>;
 
 def lower_mulv2s64 : GICombineRule<
   (defs root:$root),
@@ -291,7 +297,7 @@ def lower_mulv2s64 : GICombineRule<
 
 def build_vector_to_dup : GICombineRule<
   (defs root:$root, register_matchinfo:$matchinfo),
-  (match (wip_match_opcode G_BUILD_VECTOR):$root,
+  (match (G_BUILD_VECTOR $dst, GIVariadic<>:$unused):$root,
           [{ return matchBuildVectorToDup(*${root}, ${matchinfo}, MRI); }]),
   (apply [{ applyBuildVectorToDup(*${root}, ${matchinfo}, MRI, B); }])
 >;
@@ -321,17 +327,24 @@ def form_truncstore : GICombineRule<
 
 def fold_merge_to_zext : GICombineRule<
   (defs root:$d),
-  (match (wip_match_opcode G_MERGE_VALUES):$d,
-          [{ return matchFoldMergeToZext(*${d}, MRI); }]),
-  (apply [{ applyFoldMergeToZext(*${d}, MRI, B, Observer); }])
+  (match (G_CONSTANT $zero, 0),
+         (G_MERGE_VALUES $dst, $x, $zero):$d,
+         [{ return MRI.getType(${x}.getReg()) == LLT::scalar(32); }]),
+  (apply (G_ZEXT $dst, $x))
 >;
 
-def mutate_anyext_to_zext : GICombineRule<
+// G_ANYEXT of a scalar compare (0/1) -> G_ZEXT, to expose more KnownBits combines.
+class mutate_anyext_to_zext_cmp<Instruction cmpOpc> : GICombineRule<
   (defs root:$d),
-  (match (wip_match_opcode G_ANYEXT):$d,
-          [{ return matchMutateAnyExtToZExt(*${d}, MRI); }]),
-  (apply [{ applyMutateAnyExtToZExt(*${d}, MRI, B, Observer); }])
+  (match (cmpOpc $cmp, $p, $a, $b),
+         (G_ANYEXT $dst, $cmp):$d,
+         [{ return MRI.getType(${dst}.getReg()).isScalar(); }]),
+  (apply (G_ZEXT $dst, $cmp))
 >;
+def mutate_anyext_to_zext_icmp : mutate_anyext_to_zext_cmp<G_ICMP>;
+def mutate_anyext_to_zext_fcmp : mutate_anyext_to_zext_cmp<G_FCMP>;
+def mutate_anyext_to_zext : GICombineGroup<[mutate_anyext_to_zext_icmp,
+                                            mutate_anyext_to_zext_fcmp]>;
 
 def split_store_zero_128 : GICombineRule<
   (defs root:$d),
@@ -342,7 +355,7 @@ def split_store_zero_128 : GICombineRule<
 
 def vector_sext_inreg_to_shift : GICombineRule<
   (defs root:$d),
-  (match (wip_match_opcode G_SEXT_INREG):$d,
+  (match (G_SEXT_INREG $dst, $src, $imm):$d,
           [{ return matchVectorSextInReg(*${d}, MRI); }]),
   (apply [{ applyVectorSextInReg(*${d}, MRI, B, Observer); }])
 >;
@@ -375,7 +388,7 @@ def trunc_or_to_addhn : GICombineRule <
 // Combines Mul(And(Srl(X, 15), 0x10001), 0xffff) into CMLTz
 def combine_mul_cmlt : GICombineRule<
   (defs root:$root, register_matchinfo:$matchinfo),
-  (match (wip_match_opcode G_MUL):$root,
+  (match (G_MUL $dst, $src1, $src2):$root,
         [{ return matchCombineMulCMLT(*${root}, MRI, ${matchinfo}); }]),
   (apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }])
 >;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index 6ba25ac4ffa0a..c0f54356a4fca 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -257,54 +257,6 @@ void applyAArch64MulConstCombine(
   MI.eraseFromParent();
 }
 
-/// Try to fold a G_MERGE_VALUES of 2 s32 sources, where the second source
-/// is a zero, into a G_ZEXT of the first.
-bool matchFoldMergeToZext(MachineInstr &MI, MachineRegisterInfo &MRI) {
-  auto &Merge = cast<GMerge>(MI);
-  LLT SrcTy = MRI.getType(Merge.getSourceReg(0));
-  if (SrcTy != LLT::scalar(32) || Merge.getNumSources() != 2)
-    return false;
-  return mi_match(Merge.getSourceReg(1), MRI, m_SpecificICst(0));
-}
-
-void applyFoldMergeToZext(MachineInstr &MI, MachineRegisterInfo &MRI,
-                          MachineIRBuilder &B, GISelChangeObserver &Observer) {
-  // Mutate %d(s64) = G_MERGE_VALUES %a(s32), 0(s32)
-  //  ->
-  // %d(s64) = G_ZEXT %a(s32)
-  Observer.changingInstr(MI);
-  MI.setDesc(B.getTII().get(TargetOpcode::G_ZEXT));
-  MI.removeOperand(2);
-  Observer.changedInstr(MI);
-}
-
-/// \returns True if a G_ANYEXT instruction \p MI should be mutated to a G_ZEXT
-/// instruction.
-bool matchMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI) {
-  // If this is coming from a scalar compare then we can use a G_ZEXT instead of
-  // a G_ANYEXT:
-  //
-  // %cmp:_(s32) = G_[I|F]CMP ... <-- produces 0/1.
-  // %ext:_(s64) = G_ANYEXT %cmp(s32)
-  //
-  // By doing this, we can leverage more KnownBits combines.
-  assert(MI.getOpcode() == TargetOpcode::G_ANYEXT);
-  Register Dst = MI.getOperand(0).getReg();
-  Register Src = MI.getOperand(1).getReg();
-  return MRI.getType(Dst).isScalar() &&
-         mi_match(Src, MRI,
-                  m_any_of(m_GICmp(m_Pred(), m_Reg(), m_Reg()),
-                           m_GFCmp(m_Pred(), m_Reg(), m_Reg())));
-}
-
-void applyMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI,
-                             MachineIRBuilder &B,
-                             GISelChangeObserver &Observer) {
-  Observer.changingInstr(MI);
-  MI.setDesc(B.getTII().get(TargetOpcode::G_ZEXT));
-  Observer.changedInstr(MI);
-}
-
 /// Match a 128b store of zero and split it into two 64 bit stores, for
 /// size/performance reasons.
 bool matchSplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI) {
@@ -608,21 +560,6 @@ static bool matchSubAddMulReassoc(Register Mul1, Register Mul2, Register Sub,
   return true;
 }
 
-static void applySubAddMulReassoc(MachineInstr &MI, MachineInstr &Sub,
-                                  MachineRegisterInfo &MRI, MachineIRBuilder &B,
-                                  GISelChangeObserver &Observer) {
-  Register Src = MI.getOperand(1).getReg();
-  Register Tmp = MI.getOperand(2).getReg();
-  Register Mul1 = Sub.getOperand(1).getReg();
-  Register Mul2 = Sub.getOperand(2).getReg();
-  Observer.changingInstr(MI);
-  B.buildInstr(AArch64::G_SUB, {Tmp}, {Src, Mul1});
-  MI.getOperand(1).setReg(Tmp);
-  MI.getOperand(2).setReg(Mul2);
-  Sub.eraseFromParent();
-  Observer.changedInstr(MI);
-}
-
 class AArch64PostLegalizerCombinerImpl : public Combiner {
 protected:
   const CombinerHelper Helper;
diff --git a/llvm/lib/Target/SPIRV/SPIRVCombine.td b/llvm/lib/Target/SPIRV/SPIRVCombine.td
index 44512229a8ff1..cfab5ec26b4e1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCombine.td
+++ b/llvm/lib/Target/SPIRV/SPIRVCombine.td
@@ -8,23 +8,24 @@
 include "llvm/Target/GlobalISel/Combine.td"
 
 
+// length(x - y) -> distance(x, y)
 def vector_length_sub_to_distance_lowering : GICombineRule <
   (defs root:$root),
-  (match (wip_match_opcode G_INTRINSIC):$root,
-          [{ return Helper.matchLengthToDistance(*${root}); }]),
-  (apply [{ Helper.applySPIRVDistance(*${root}); }])
+  (match (G_FSUB $sub, $x, $y),
+         (int_spv_length $dst, $sub):$root),
+  (apply (int_spv_distance $dst, $x, $y))
 >;
 
 def vector_fdiv_length_to_normalize_lowering : GICombineRule <
   (defs root:$root),
-  (match (wip_match_opcode G_FDIV):$root,
+  (match (G_FDIV $dst, $src1, $src2):$root,
           [{ return Helper.matchFDivToNormalize(*${root}); }]),
   (apply [{ Helper.applySPIRVNormalize(*${root}); }])
 >;
 
 def vector_select_to_faceforward_lowering : GICombineRule <
   (defs root:$root),
-  (match (wip_match_opcode G_SELECT):$root,
+  (match (G_SELECT $dst, $cond, $t, $f):$root,
           [{ return Helper.matchSelectToFaceForward(*${root}); }]),
   (apply [{ Helper.applySPIRVFaceForward(*${root}); }])
 >;
@@ -32,17 +33,13 @@ def vector_select_to_faceforward_lowering : GICombineRule <
 def matrix_transpose_lowering
     : GICombineRule<
           (defs root:$root),
-          (match (wip_match_opcode G_INTRINSIC):$root,
-              [{ return mi_match(*${root}, MRI,
-                                 m_GIntrinsic<Intrinsic::matrix_transpose>()); }]),
+          (match (int_matrix_transpose $dst, $matrix, $rows, $cols):$root),
           (apply [{ Helper.applyMatrixTranspose(*${root}); }])>;
 
 def matrix_multiply_lowering
     : GICombineRule<
           (defs root:$root),
-          (match (wip_match_opcode G_INTRINSIC):$root,
-              [{ return mi_match(*${root}, MRI,
-                                 m_GIntrinsic<Intrinsic::matrix_multiply>()); }]),
+          (match (int_matrix_multiply $dst, $a, $b, $m, $n, $k):$root),
           (apply [{ Helper.applyMatrixMultiply(*${root}); }])>;
 
 def SPIRVPreLegalizerCombiner
diff --git a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp
index 63fe8688359fd..d8fbfb87a0115 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp
@@ -24,39 +24,6 @@ SPIRVCombinerHelper::SPIRVCombinerHelper(
     const SPIRVSubtarget &STI)
     : CombinerHelper(Observer, B, IsPreLegalize, VT, MDT, LI), STI(STI) {}
 
-/// This match is part of a combine that
-/// rewrites length(X - Y) to distance(X, Y)
-///   (f32 (g_intrinsic length
-///           (g_fsub (vXf32 X) (vXf32 Y))))
-/// ->
-///   (f32 (g_intrinsic distance
-///           (vXf32 X) (vXf32 Y)))
-///
-bool SPIRVCombinerHelper::matchLengthToDistance(MachineInstr &MI) const {
-  if (!mi_match(MI, MRI, m_GIntrinsic<Intrinsic::spv_length>()))
-    return false;
-
-  // First operand of MI is `G_INTRINSIC` so start at operand 2.
-  Register SubReg = MI.getOperand(2).getReg();
-  return mi_match(SubReg, MRI, m_GFSub(m_Reg(), m_Reg()));
-}
-
-void SPIRVCombinerHelper::applySPIRVDistance(MachineInstr &MI) const {
-  // Extract the operands for X and Y from the match criteria.
-  Register SubDestReg = MI.getOperand(2).getReg();
-  MachineInstr *SubInstr = MRI.getVRegDef(SubDestReg);
-  Register SubOperand1 = SubInstr->getOperand(1).getReg();
-  Register SubOperand2 = SubInstr->getOperand(2).getReg();
-  Register ResultReg = MI.getOperand(0).getReg();
-
-  Builder.setInstrAndDebugLoc(MI);
-  Builder.buildIntrinsic(Intrinsic::spv_distance, ResultReg)
-      .addUse(SubOperand1)
-      .addUse(SubOperand2);
-
-  MI.eraseFromParent();
-}
-
 /// This match is part of a combine that
 /// rewrites X / length(X) to normalize(X)
 ///   (vXf32 (g_fdiv
diff --git a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h
index af9bcd7347980..3912f4608cf2f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h
+++ b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h
@@ -29,8 +29,6 @@ class SPIRVCombinerHelper : public CombinerHelper {
                       MachineDominatorTree *MDT, const LegalizerInfo *LI,
                       const SPIRVSubtarget &STI);
 
-  bool matchLengthToDistance(MachineInstr &MI) const;
-  void applySPIRVDistance(MachineInstr &MI) const;
   bool matchFDivToNormalize(MachineInstr &MI) const;
   void applySPIRVNormalize(MachineInstr &MI) const;
   bool matchSelectToFaceForward(MachineInstr &MI) const;
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp
index 1e1fbe75f53c2..cc4ee20ceee7d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Analysis.h"
+#include "llvm/IR/IntrinsicsSPIRV.h"
 
 #define GET_GICOMBINER_DEPS
 #include "SPIRVGenPreLegalizeGICombiner.inc"
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-anyext-to-zext.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-anyext-to-zext.mir
index 69d2f9f0cd3bb..bab7c192da084 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-anyext-to-zext.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-anyext-to-zext.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner --aarch64postlegalizercombiner-only-enable-rule="mutate_anyext_to_zext" -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner --aarch64postlegalizercombiner-only-enable-rule="mutate_anyext_to_zext_icmp,mutate_anyext_to_zext_fcmp" -verify-machineinstrs %s -o - | FileCheck %s
 # REQUIRES: asserts
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll b/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
index 1fa1cf135051a..138e0d0588aa7 100644
--- a/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
@@ -24,17 +24,15 @@ define <4 x i8> @test_varidx_extract_v8s8(<8 x i8> %x, i32 %idx) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    sub sp, sp, #16
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT:    mov w9, w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov w9, w0
 ; CHECK-GI-NEXT:    mov b1, v0.b[1]
 ; CHECK-GI-NEXT:    add x8, sp, #8
-; CHECK-GI-NEXT:    and x9, x9, #0x7
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
+; CHECK-GI-NEXT:    and x9, x9, #0x7
 ; CHECK-GI-NEXT:    mov b2, v0.b[2]
-; CHECK-GI-NEXT:    lsl x10, x9, #1
-; CHECK-GI-NEXT:    mov b0, v0.b[3]
-; CHECK-GI-NEXT:    sub x9, x10, x9
 ; CHECK-GI-NEXT:    ldrb w8, [x8, x9]
+; CHECK-GI-NEXT:    mov b0, v0.b[3]
 ; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    fmov s1, w8
 ; CHECK-GI-NEXT:    fmov w8, s2
@@ -85,8 +83,6 @@ define <8 x i8> @test_varidx_extract_v16s8(<16 x i8> %x, i32 %idx) {
 ; CHECK-GI-NEXT:    mov x8, sp
 ; CHECK-GI-NEXT:    str q0, [sp]
 ; CHECK-GI-NEXT:    and x9, x9, #0xf
-; CHECK-GI-NEXT:    lsl x10, x9, #1
-; CHECK-GI-NEXT:    sub x9, x10, x9
 ; CHECK-GI-NEXT:    ldr b1, [x8, x9]
 ; CHECK-GI-NEXT:    mov v1.b[1], v0.b[1]
 ; CHECK-GI-NEXT:    mov v1.b[2], v0.b[2]
diff --git a/llvm/test/CodeGen/AArch64/insertextract.ll b/llvm/test/CodeGen/AArch64/insertextract.ll
index 8b65000bd1a1e..ab28e11d4138a 100644
--- a/llvm/test/CodeGen/AArch64/insertextract.ll
+++ b/llvm/test/CodeGen/AArch64/insertextract.ll
@@ -732,12 +732,10 @@ define <32 x i8> @insert_v32i8_c(<32 x i8> %a, i8 %b, i32 %c) {
 ; CHECK-GI-NEXT:    .cfi_offset w30, -8
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    mov w8, w1
-; CHECK-GI-NEXT:    mov x10, sp
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    stp q0, q1, [sp]
 ; CHECK-GI-NEXT:    and x8, x8, #0x1f
-; CHECK-GI-NEXT:    lsl x9, x8, #1
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    strb w0, [x10, x8]
+; CHECK-GI-NEXT:    strb w0, [x9, x8]
 ; CHECK-GI-NEXT:    ldp q0, q1, [sp]
 ; CHECK-GI-NEXT:    mov sp, x29
 ; CHECK-GI-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
@@ -2135,8 +2133,6 @@ define i8 @extract_v8i8_c(<8 x i8> %a, i32 %c) {
 ; CHECK-GI-NEXT:    add x8, sp, #8
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
 ; CHECK-GI-NEXT:    and x9, x9, #0x7
-; CHECK-GI-NEXT:    lsl x10, x9, #1
-; CHECK-GI-NEXT:    sub x9, x10, x9
 ; CHECK-GI-NEXT:    ldrb w0, [x8, x9]
 ; CHECK-GI-NEXT:    add sp, sp, #16
 ; CHECK-GI-NEXT:    ret
@@ -2186,8 +2182,6 @@ define i8 @extract_v16i8_c(<16 x i8> %a, i32 %c) {
 ; CHECK-GI-NEXT:    mov x8, sp
 ; CHECK-GI-NEXT:    str q0, [sp]
 ; CHECK-GI-NEXT:    and x9, x9, #0xf
-; CHECK-GI-NEXT:    lsl x10, x9, #1
-; CHECK-GI-NEXT:    sub x9, x10, x9
 ; CHECK-GI-NEXT:    ldrb w0, [x8, x9]
 ; CHECK-GI-NEXT:    add sp, sp, #16
 ; CHECK-GI-NEXT:    ret
@@ -2239,11 +2233,9 @@ define i8 @extract_v32i8_c(<32 x i8> %a, i32 %c) {
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    mov w8, w0
 ; CHECK-GI-NEXT:    stp q0, q1, [sp]
-; CHECK-GI-NEXT:    mov x10, sp
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    and x8, x8, #0x1f
-; CHECK-GI-NEXT:    lsl x9, x8, #1
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    ldrb w0, [x10, x8]
+; CHECK-GI-NEXT:    ldrb w0, [x9, x8]
 ; CHECK-GI-NEXT:    mov sp, x29
 ; CHECK-GI-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll b/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
index 93c22573c0511..30949de41ae1a 100644
--- a/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
@@ -18,119 +18,87 @@ define <16 x i8> @shuffle16_with_and_mask(<16 x i8> %src, <16 x i8> %mask) {
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 272
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    movi.16b v2, #15
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    str q0, [sp]
 ; CHECK-GI-NEXT:    ldr x29, [sp, #256] // 8-byte Reload
 ; CHECK-GI-NEXT:    and.16b v2, v1, v2
-; CHECK-GI-NEXT:    umov.b w9, v2[0]
+; CHECK-GI-NEXT:    umov.b w8, v2[0]
+; CHECK-GI-NEXT:    ldr b1, [x9, x8]
 ; CHECK-GI-NEXT:    umov.b w8, v2[1]
-; CHECK-GI-NEXT:    lsl x10, x9, #1
-; CHECK-GI-NEXT:    sub x9, x10, x9
-; CHECK-GI-NEXT:    mov x10, sp
-; CHECK-GI-NEXT:    ldr b1, [x10, x9]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[2]
+; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    str q0, [sp, #16]
 ; CHECK-GI-NEXT:    mov.b v1[0], v1[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #16
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[3]
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[2]
+; CHECK-GI-NEXT:    add x9, sp, #32
 ; CHECK-GI-NEXT:    str q0, [sp, #32]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
 ; CHECK-GI-NEXT:    mov.b v1[1], v3[0]
-; CHECK-GI-NEXT:    sub x8, x8, x9
-; CHECK-GI-NEXT:    add x9, sp, #32
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[4]
+; CHECK-GI-NEXT:    umov.b w8, v2[3]
+; CHECK-GI-NEXT:    add x9, sp, #48
 ; CHECK-GI-NEXT:    str q0, [sp, #48]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #48
 ; CHECK-GI-NEXT:    mov.b v1[2], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[5]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #64]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[4]
 ; CHECK-GI-NEXT:    add x9, sp, #64
+; CHECK-GI-NEXT:    str q0, [sp, #64]
 ; CHECK-GI-NEXT:    mov.b v1[3], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[6]
+; CHECK-GI-NEXT:    umov.b w8, v2[5]
+; CHECK-GI-NEXT:    add x9, sp, #80
 ; CHECK-GI-NEXT:    str q0, [sp, #80]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #80
 ; CHECK-GI-NEXT:    mov.b v1[4], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[7]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #96]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[6]
 ; CHECK-GI-NEXT:    add x9, sp, #96
+; CHECK-GI-NEXT:    str q0, [sp, #96]
 ; CHECK-GI-NEXT:    mov.b v1[5], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[8]
+; CHECK-GI-NEXT:    umov.b w8, v2[7]
+; CHECK-GI-NEXT:    add x9, sp, #112
 ; CHECK-GI-NEXT:    str q0, [sp, #112]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #112
 ; CHECK-GI-NEXT:    mov.b v1[6], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[9]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #128]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[8]
 ; CHECK-GI-NEXT:    add x9, sp, #128
+; CHECK-GI-NEXT:    str q0, [sp, #128]
 ; CHECK-GI-NEXT:    mov.b v1[7], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[10]
+; CHECK-GI-NEXT:    umov.b w8, v2[9]
+; CHECK-GI-NEXT:    add x9, sp, #144
 ; CHECK-GI-NEXT:    str q0, [sp, #144]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #144
 ; CHECK-GI-NEXT:    mov.b v1[8], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[11]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #160]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[10]
 ; CHECK-GI-NEXT:    add x9, sp, #160
+; CHECK-GI-NEXT:    str q0, [sp, #160]
 ; CHECK-GI-NEXT:    mov.b v1[9], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[12]
+; CHECK-GI-NEXT:    umov.b w8, v2[11]
+; CHECK-GI-NEXT:    add x9, sp, #176
 ; CHECK-GI-NEXT:    str q0, [sp, #176]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #176
 ; CHECK-GI-NEXT:    mov.b v1[10], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[13]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #192]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[12]
 ; CHECK-GI-NEXT:    add x9, sp, #192
+; CHECK-GI-NEXT:    str q0, [sp, #192]
 ; CHECK-GI-NEXT:    mov.b v1[11], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[14]
+; CHECK-GI-NEXT:    umov.b w8, v2[13]
+; CHECK-GI-NEXT:    add x9, sp, #208
 ; CHECK-GI-NEXT:    str q0, [sp, #208]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #208
 ; CHECK-GI-NEXT:    mov.b v1[12], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[15]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #224]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[14]
 ; CHECK-GI-NEXT:    add x9, sp, #224
+; CHECK-GI-NEXT:    str q0, [sp, #224]
 ; CHECK-GI-NEXT:    mov.b v1[13], v3[0]
-; CHECK-GI-NEXT:    ldr b2, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[15]
 ; CHECK-GI-NEXT:    add x9, sp, #240
 ; CHECK-GI-NEXT:    str q0, [sp, #240]
-; CHECK-GI-NEXT:    sub x8, x8, x10
+; CHECK-GI-NEXT:    mov.b v1[14], v3[0]
 ; CHECK-GI-NEXT:    ldr b0, [x9, x8]
-; CHECK-GI-NEXT:    mov.b v1[14], v2[0]
 ; CHECK-GI-NEXT:    mov.b v1[15], v0[0]
 ; CHECK-GI-NEXT:    mov.16b v0, v1
 ; CHECK-GI-NEXT:    add sp, sp, #272
@@ -204,62 +172,47 @@ define <8 x i8> @shuffle8_with_and_mask(<8 x i8> %src, <8 x i8> %mask) {
 ; CHECK-GI-NEXT:    sub sp, sp, #64
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-GI-NEXT:    movi.8b v2, #7
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    str d0, [sp]
 ; CHECK-GI-NEXT:    and.8b v2, v1, v2
 ; CHECK-GI-NEXT:    umov.b w8, v2[0]
-; CHECK-GI-NEXT:    umov.b w10, v2[1]
-; CHECK-GI-NEXT:    lsl x9, x8, #1
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    ldr b1, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[2]
+; CHECK-GI-NEXT:    umov.b w8, v2[1]
+; CHECK-GI-NEXT:    add x9, sp, #8
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
 ; CHECK-GI-NEXT:    mov.b v1[0], v1[0]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #8
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[3]
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[2]
+; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    str d0, [sp, #16]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
 ; CHECK-GI-NEXT:    mov.b v1[1], v3[0]
-; CHECK-GI-NEXT:    sub x8, x8, x9
-; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[4]
+; CHECK-GI-NEXT:    umov.b w8, v2[3]
+; CHECK-GI-NEXT:    add x9, sp, #24
 ; CHECK-GI-NEXT:    str d0, [sp, #24]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #24
 ; CHECK-GI-NEXT:    mov.b v1[2], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[5]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str d0, [sp, #32]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[4]
 ; CHECK-GI-NEXT:    add x9, sp, #32
+; CHECK-GI-NEXT:    str d0, [sp, #32]
 ; CHECK-GI-NEXT:    mov.b v1[3], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[6]
+; CHECK-GI-NEXT:    umov.b w8, v2[5]
+; CHECK-GI-NEXT:    add x9, sp, #40
 ; CHECK-GI-NEXT:    str d0, [sp, #40]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #40
 ; CHECK-GI-NEXT:    mov.b v1[4], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[7]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str d0, [sp, #48]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[6]
+; CHECK-GI-NEXT:    mov b2, v2[7]
 ; CHECK-GI-NEXT:    add x9, sp, #48
+; CHECK-GI-NEXT:    str d0, [sp, #48]
 ; CHECK-GI-NEXT:    mov.b v1[5], v3[0]
-; CHECK-GI-NEXT:    ldr b2, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    fmov w8, s2
 ; CHECK-GI-NEXT:    add x9, sp, #56
 ; CHECK-GI-NEXT:    str d0, [sp, #56]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x8, x9, x8
-; CHECK-GI-NEXT:    mov.b v1[6], v2[0]
+; CHECK-GI-NEXT:    mov.b v1[6], v3[0]
+; CHECK-GI-NEXT:    add x8, x9, w8, uxtb
 ; CHECK-GI-NEXT:    ld1.b { v1 }[7], [x8]
 ; CHECK-GI-NEXT:    fmov d0, d1
 ; CHECK-GI-NEXT:    add sp, sp, #64
@@ -308,63 +261,48 @@ define <8 x i8> @shuffle8_with_and_mask_different_constants(<8 x i8> %src, <8 x
 ; CHECK-GI-NEXT:    sub sp, sp, #64
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-GI-NEXT:    adrp x8, .LCPI2_0
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    str d0, [sp]
 ; CHECK-GI-NEXT:    ldr d2, [x8, :lo12:.LCPI2_0]
 ; CHECK-GI-NEXT:    and.8b v2, v1, v2
 ; CHECK-GI-NEXT:    umov.b w8, v2[0]
-; CHECK-GI-NEXT:    umov.b w10, v2[1]
-; CHECK-GI-NEXT:    lsl x9, x8, #1
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    ldr b1, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[2]
+; CHECK-GI-NEXT:    umov.b w8, v2[1]
+; CHECK-GI-NEXT:    add x9, sp, #8
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
 ; CHECK-GI-NEXT:    mov.b v1[0], v1[0]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #8
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[3]
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[2]
+; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    str d0, [sp, #16]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
 ; CHECK-GI-NEXT:    mov.b v1[1], v3[0]
-; CHECK-GI-NEXT:    sub x8, x8, x9
-; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[4]
+; CHECK-GI-NEXT:    umov.b w8, v2[3]
+; CHECK-GI-NEXT:    add x9, sp, #24
 ; CHECK-GI-NEXT:    str d0, [sp, #24]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #24
 ; CHECK-GI-NEXT:    mov.b v1[2], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[5]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str d0, [sp, #32]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[4]
 ; CHECK-GI-NEXT:    add x9, sp, #32
+; CHECK-GI-NEXT:    str d0, [sp, #32]
 ; CHECK-GI-NEXT:    mov.b v1[3], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[6]
+; CHECK-GI-NEXT:    umov.b w8, v2[5]
+; CHECK-GI-NEXT:    add x9, sp, #40
 ; CHECK-GI-NEXT:    str d0, [sp, #40]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #40
 ; CHECK-GI-NEXT:    mov.b v1[4], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[7]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str d0, [sp, #48]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[6]
+; CHECK-GI-NEXT:    mov b2, v2[7]
 ; CHECK-GI-NEXT:    add x9, sp, #48
+; CHECK-GI-NEXT:    str d0, [sp, #48]
 ; CHECK-GI-NEXT:    mov.b v1[5], v3[0]
-; CHECK-GI-NEXT:    ldr b2, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    fmov w8, s2
 ; CHECK-GI-NEXT:    add x9, sp, #56
 ; CHECK-GI-NEXT:    str d0, [sp, #56]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x8, x9, x8
-; CHECK-GI-NEXT:    mov.b v1[6], v2[0]
+; CHECK-GI-NEXT:    mov.b v1[6], v3[0]
+; CHECK-GI-NEXT:    add x8, x9, w8, uxtb
 ; CHECK-GI-NEXT:    ld1.b { v1 }[7], [x8]
 ; CHECK-GI-NEXT:    fmov d0, d1
 ; CHECK-GI-NEXT:    add sp, sp, #64
@@ -412,68 +350,52 @@ define <8 x i8> @shuffle8_with_mask(<8 x i8> %src, <8 x i8> %mask) {
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-GI-NEXT:    umov.b w9, v1[0]
-; CHECK-GI-NEXT:    umov.b w10, v1[1]
 ; CHECK-GI-NEXT:    mov x8, sp
 ; CHECK-GI-NEXT:    str d0, [sp]
+; CHECK-GI-NEXT:    umov.b w10, v1[1]
 ; CHECK-GI-NEXT:    and x9, x9, #0x7
-; CHECK-GI-NEXT:    lsl x11, x9, #1
-; CHECK-GI-NEXT:    sub x9, x11, x9
 ; CHECK-GI-NEXT:    ldr b2, [x8, x9]
-; CHECK-GI-NEXT:    and x8, x10, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[2]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    add x8, sp, #8
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[2]
 ; CHECK-GI-NEXT:    mov.b v2[0], v2[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #8
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[3]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #16
 ; CHECK-GI-NEXT:    str d0, [sp, #16]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[3]
 ; CHECK-GI-NEXT:    mov.b v2[1], v3[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #16
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[4]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #24
 ; CHECK-GI-NEXT:    str d0, [sp, #24]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[4]
 ; CHECK-GI-NEXT:    mov.b v2[2], v3[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #24
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[5]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #32
 ; CHECK-GI-NEXT:    str d0, [sp, #32]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[5]
 ; CHECK-GI-NEXT:    mov.b v2[3], v3[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #32
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[6]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #40
 ; CHECK-GI-NEXT:    str d0, [sp, #40]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[6]
 ; CHECK-GI-NEXT:    mov.b v2[4], v3[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #40
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w10, v1[7]
-; CHECK-GI-NEXT:    lsl x9, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #48
 ; CHECK-GI-NEXT:    str d0, [sp, #48]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
 ; CHECK-GI-NEXT:    mov.b v2[5], v3[0]
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    add x9, sp, #48
-; CHECK-GI-NEXT:    ldr b1, [x9, x8]
-; CHECK-GI-NEXT:    and x8, x10, #0x7
-; CHECK-GI-NEXT:    add x10, sp, #56
-; CHECK-GI-NEXT:    lsl x9, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    umov.b w8, v1[7]
+; CHECK-GI-NEXT:    add x9, sp, #56
 ; CHECK-GI-NEXT:    str d0, [sp, #56]
-; CHECK-GI-NEXT:    mov.b v2[6], v1[0]
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    add x8, x10, x8
+; CHECK-GI-NEXT:    mov.b v2[6], v3[0]
+; CHECK-GI-NEXT:    and x8, x8, #0x7
+; CHECK-GI-NEXT:    add x8, x9, x8
 ; CHECK-GI-NEXT:    ld1.b { v2 }[7], [x8]
 ; CHECK-GI-NEXT:    fmov d0, d2
 ; CHECK-GI-NEXT:    add sp, sp, #64
@@ -555,69 +477,53 @@ define <8 x i8> @no_shuffle_only_some_and_constants(<8 x i8> %src, <8 x i8> %mas
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-GI-NEXT:    umov.b w9, v1[0]
-; CHECK-GI-NEXT:    umov.b w10, v1[1]
 ; CHECK-GI-NEXT:    mov x8, sp
 ; CHECK-GI-NEXT:    str d0, [sp]
+; CHECK-GI-NEXT:    umov.b w10, v1[1]
 ; CHECK-GI-NEXT:    and w9, w9, #0x7
 ; CHECK-GI-NEXT:    and x9, x9, #0x7
-; CHECK-GI-NEXT:    lsl x11, x9, #1
-; CHECK-GI-NEXT:    sub x9, x11, x9
 ; CHECK-GI-NEXT:    ldr b2, [x8, x9]
-; CHECK-GI-NEXT:    and x8, x10, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[2]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    add x8, sp, #8
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[2]
 ; CHECK-GI-NEXT:    mov.b v2[0], v2[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #8
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[3]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #16
 ; CHECK-GI-NEXT:    str d0, [sp, #16]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[3]
 ; CHECK-GI-NEXT:    mov.b v2[1], v3[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #16
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[4]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #24
 ; CHECK-GI-NEXT:    str d0, [sp, #24]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[4]
 ; CHECK-GI-NEXT:    mov.b v2[2], v3[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #24
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[5]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #32
 ; CHECK-GI-NEXT:    str d0, [sp, #32]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[5]
 ; CHECK-GI-NEXT:    mov.b v2[3], v3[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #32
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w9, v1[6]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #40
 ; CHECK-GI-NEXT:    str d0, [sp, #40]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
+; CHECK-GI-NEXT:    umov.b w10, v1[6]
 ; CHECK-GI-NEXT:    mov.b v2[4], v3[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #40
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    and x8, x9, #0x7
-; CHECK-GI-NEXT:    umov.b w10, v1[7]
-; CHECK-GI-NEXT:    lsl x9, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    add x8, sp, #48
 ; CHECK-GI-NEXT:    str d0, [sp, #48]
+; CHECK-GI-NEXT:    and x9, x10, #0x7
 ; CHECK-GI-NEXT:    mov.b v2[5], v3[0]
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    add x9, sp, #48
-; CHECK-GI-NEXT:    ldr b1, [x9, x8]
-; CHECK-GI-NEXT:    and x8, x10, #0x7
-; CHECK-GI-NEXT:    add x10, sp, #56
-; CHECK-GI-NEXT:    lsl x9, x8, #1
+; CHECK-GI-NEXT:    ldr b3, [x8, x9]
+; CHECK-GI-NEXT:    umov.b w8, v1[7]
+; CHECK-GI-NEXT:    add x9, sp, #56
 ; CHECK-GI-NEXT:    str d0, [sp, #56]
-; CHECK-GI-NEXT:    mov.b v2[6], v1[0]
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    add x8, x10, x8
+; CHECK-GI-NEXT:    mov.b v2[6], v3[0]
+; CHECK-GI-NEXT:    and x8, x8, #0x7
+; CHECK-GI-NEXT:    add x8, x9, x8
 ; CHECK-GI-NEXT:    ld1.b { v2 }[7], [x8]
 ; CHECK-GI-NEXT:    fmov d0, d2
 ; CHECK-GI-NEXT:    add sp, sp, #64
@@ -735,120 +641,88 @@ define <16 x i8> @no_shuffle_with_two_mask_sources(<16 x i8> %src, <16 x i8> %ma
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 272
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    movi.16b v4, #15
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    str q0, [sp]
 ; CHECK-GI-NEXT:    ldr x29, [sp, #256] // 8-byte Reload
 ; CHECK-GI-NEXT:    and.16b v3, v1, v4
 ; CHECK-GI-NEXT:    and.16b v2, v2, v4
-; CHECK-GI-NEXT:    umov.b w9, v3[0]
+; CHECK-GI-NEXT:    umov.b w8, v3[0]
+; CHECK-GI-NEXT:    ldr b1, [x9, x8]
 ; CHECK-GI-NEXT:    umov.b w8, v2[1]
-; CHECK-GI-NEXT:    lsl x10, x9, #1
-; CHECK-GI-NEXT:    sub x9, x10, x9
-; CHECK-GI-NEXT:    mov x10, sp
-; CHECK-GI-NEXT:    ldr b1, [x10, x9]
-; CHECK-GI-NEXT:    lsl x10, x8, #1
-; CHECK-GI-NEXT:    umov.b w9, v3[2]
+; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    str q0, [sp, #16]
 ; CHECK-GI-NEXT:    mov.b v1[0], v1[0]
-; CHECK-GI-NEXT:    sub x8, x10, x8
-; CHECK-GI-NEXT:    add x10, sp, #16
-; CHECK-GI-NEXT:    ldr b4, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[3]
+; CHECK-GI-NEXT:    ldr b4, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v3[2]
+; CHECK-GI-NEXT:    add x9, sp, #32
 ; CHECK-GI-NEXT:    str q0, [sp, #32]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
 ; CHECK-GI-NEXT:    mov.b v1[1], v4[0]
-; CHECK-GI-NEXT:    sub x8, x8, x9
-; CHECK-GI-NEXT:    add x9, sp, #32
 ; CHECK-GI-NEXT:    ldr b4, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v3[4]
+; CHECK-GI-NEXT:    umov.b w8, v2[3]
+; CHECK-GI-NEXT:    add x9, sp, #48
 ; CHECK-GI-NEXT:    str q0, [sp, #48]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #48
 ; CHECK-GI-NEXT:    mov.b v1[2], v4[0]
-; CHECK-GI-NEXT:    ldr b4, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[5]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #64]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b4, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v3[4]
 ; CHECK-GI-NEXT:    add x9, sp, #64
+; CHECK-GI-NEXT:    str q0, [sp, #64]
 ; CHECK-GI-NEXT:    mov.b v1[3], v4[0]
 ; CHECK-GI-NEXT:    ldr b4, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v3[6]
+; CHECK-GI-NEXT:    umov.b w8, v2[5]
+; CHECK-GI-NEXT:    add x9, sp, #80
 ; CHECK-GI-NEXT:    str q0, [sp, #80]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #80
 ; CHECK-GI-NEXT:    mov.b v1[4], v4[0]
-; CHECK-GI-NEXT:    ldr b4, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[7]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #96]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b4, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v3[6]
 ; CHECK-GI-NEXT:    add x9, sp, #96
+; CHECK-GI-NEXT:    str q0, [sp, #96]
 ; CHECK-GI-NEXT:    mov.b v1[5], v4[0]
 ; CHECK-GI-NEXT:    ldr b4, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v3[8]
+; CHECK-GI-NEXT:    umov.b w8, v2[7]
+; CHECK-GI-NEXT:    add x9, sp, #112
 ; CHECK-GI-NEXT:    str q0, [sp, #112]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #112
 ; CHECK-GI-NEXT:    mov.b v1[6], v4[0]
-; CHECK-GI-NEXT:    ldr b4, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[9]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #128]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b4, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v3[8]
 ; CHECK-GI-NEXT:    add x9, sp, #128
+; CHECK-GI-NEXT:    str q0, [sp, #128]
 ; CHECK-GI-NEXT:    mov.b v1[7], v4[0]
 ; CHECK-GI-NEXT:    ldr b4, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v3[10]
+; CHECK-GI-NEXT:    umov.b w8, v2[9]
+; CHECK-GI-NEXT:    add x9, sp, #144
 ; CHECK-GI-NEXT:    str q0, [sp, #144]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #144
 ; CHECK-GI-NEXT:    mov.b v1[8], v4[0]
-; CHECK-GI-NEXT:    ldr b4, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[11]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #160]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b4, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v3[10]
 ; CHECK-GI-NEXT:    add x9, sp, #160
+; CHECK-GI-NEXT:    str q0, [sp, #160]
 ; CHECK-GI-NEXT:    mov.b v1[9], v4[0]
 ; CHECK-GI-NEXT:    ldr b4, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v3[12]
+; CHECK-GI-NEXT:    umov.b w8, v2[11]
+; CHECK-GI-NEXT:    add x9, sp, #176
 ; CHECK-GI-NEXT:    str q0, [sp, #176]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #176
 ; CHECK-GI-NEXT:    mov.b v1[10], v4[0]
-; CHECK-GI-NEXT:    ldr b4, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[13]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str q0, [sp, #192]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b4, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v3[12]
 ; CHECK-GI-NEXT:    add x9, sp, #192
+; CHECK-GI-NEXT:    str q0, [sp, #192]
 ; CHECK-GI-NEXT:    mov.b v1[11], v4[0]
 ; CHECK-GI-NEXT:    ldr b4, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v3[14]
+; CHECK-GI-NEXT:    umov.b w8, v2[13]
+; CHECK-GI-NEXT:    add x9, sp, #208
 ; CHECK-GI-NEXT:    str q0, [sp, #208]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #208
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[15]
-; CHECK-GI-NEXT:    str q0, [sp, #224]
 ; CHECK-GI-NEXT:    mov.b v1[12], v4[0]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b4, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v3[14]
 ; CHECK-GI-NEXT:    add x9, sp, #224
-; CHECK-GI-NEXT:    mov.b v1[13], v3[0]
-; CHECK-GI-NEXT:    ldr b2, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
+; CHECK-GI-NEXT:    str q0, [sp, #224]
+; CHECK-GI-NEXT:    mov.b v1[13], v4[0]
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[15]
 ; CHECK-GI-NEXT:    add x9, sp, #240
 ; CHECK-GI-NEXT:    str q0, [sp, #240]
-; CHECK-GI-NEXT:    sub x8, x8, x10
+; CHECK-GI-NEXT:    mov.b v1[14], v3[0]
 ; CHECK-GI-NEXT:    ldr b0, [x9, x8]
-; CHECK-GI-NEXT:    mov.b v1[14], v2[0]
 ; CHECK-GI-NEXT:    mov.b v1[15], v0[0]
 ; CHECK-GI-NEXT:    mov.16b v0, v1
 ; CHECK-GI-NEXT:    add sp, sp, #272
@@ -1010,34 +884,27 @@ define <8 x i8> @no_shuffle_not_enough_elements(<8 x i8> %src, <8 x i8> %mask) {
 ; CHECK-GI-NEXT:    sub sp, sp, #32
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-GI-NEXT:    movi.8b v2, #7
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    str d0, [sp]
 ; CHECK-GI-NEXT:    and.8b v2, v1, v2
 ; CHECK-GI-NEXT:    umov.b w8, v2[0]
-; CHECK-GI-NEXT:    umov.b w10, v2[1]
-; CHECK-GI-NEXT:    lsl x9, x8, #1
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    ldr b1, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[2]
+; CHECK-GI-NEXT:    umov.b w8, v2[1]
+; CHECK-GI-NEXT:    add x9, sp, #8
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
 ; CHECK-GI-NEXT:    mov.b v1[0], v1[0]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #8
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[3]
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[2]
+; CHECK-GI-NEXT:    mov b2, v2[3]
+; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    str d0, [sp, #16]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
 ; CHECK-GI-NEXT:    mov.b v1[1], v3[0]
-; CHECK-GI-NEXT:    sub x8, x8, x9
-; CHECK-GI-NEXT:    add x9, sp, #16
-; CHECK-GI-NEXT:    ldr b2, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    fmov w8, s2
 ; CHECK-GI-NEXT:    add x9, sp, #24
 ; CHECK-GI-NEXT:    str d0, [sp, #24]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    mov.b v1[2], v2[0]
-; CHECK-GI-NEXT:    add x8, x9, x8
+; CHECK-GI-NEXT:    mov.b v1[2], v3[0]
+; CHECK-GI-NEXT:    add x8, x9, w8, uxtb
 ; CHECK-GI-NEXT:    ld1.b { v1 }[3], [x8]
 ; CHECK-GI-NEXT:    fmov d0, d1
 ; CHECK-GI-NEXT:    add sp, sp, #32
@@ -1106,62 +973,47 @@ define <8 x i8> @no_shuffle_different_vector_types(<8 x i8> %src, <16 x i8> %mas
 ; CHECK-GI-NEXT:    sub sp, sp, #64
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-GI-NEXT:    movi.16b v2, #7
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    str d0, [sp]
 ; CHECK-GI-NEXT:    and.16b v2, v1, v2
 ; CHECK-GI-NEXT:    umov.b w8, v2[0]
-; CHECK-GI-NEXT:    umov.b w10, v2[1]
-; CHECK-GI-NEXT:    lsl x9, x8, #1
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    ldr b1, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[2]
+; CHECK-GI-NEXT:    umov.b w8, v2[1]
+; CHECK-GI-NEXT:    add x9, sp, #8
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
 ; CHECK-GI-NEXT:    mov.b v1[0], v1[0]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #8
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[3]
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[2]
+; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    str d0, [sp, #16]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
 ; CHECK-GI-NEXT:    mov.b v1[1], v3[0]
-; CHECK-GI-NEXT:    sub x8, x8, x9
-; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[4]
+; CHECK-GI-NEXT:    umov.b w8, v2[3]
+; CHECK-GI-NEXT:    add x9, sp, #24
 ; CHECK-GI-NEXT:    str d0, [sp, #24]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #24
 ; CHECK-GI-NEXT:    mov.b v1[2], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[5]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str d0, [sp, #32]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[4]
 ; CHECK-GI-NEXT:    add x9, sp, #32
+; CHECK-GI-NEXT:    str d0, [sp, #32]
 ; CHECK-GI-NEXT:    mov.b v1[3], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[6]
+; CHECK-GI-NEXT:    umov.b w8, v2[5]
+; CHECK-GI-NEXT:    add x9, sp, #40
 ; CHECK-GI-NEXT:    str d0, [sp, #40]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #40
 ; CHECK-GI-NEXT:    mov.b v1[4], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[7]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str d0, [sp, #48]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[6]
+; CHECK-GI-NEXT:    mov b2, v2[7]
 ; CHECK-GI-NEXT:    add x9, sp, #48
+; CHECK-GI-NEXT:    str d0, [sp, #48]
 ; CHECK-GI-NEXT:    mov.b v1[5], v3[0]
-; CHECK-GI-NEXT:    ldr b2, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    fmov w8, s2
 ; CHECK-GI-NEXT:    add x9, sp, #56
 ; CHECK-GI-NEXT:    str d0, [sp, #56]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x8, x9, x8
-; CHECK-GI-NEXT:    mov.b v1[6], v2[0]
+; CHECK-GI-NEXT:    mov.b v1[6], v3[0]
+; CHECK-GI-NEXT:    add x8, x9, w8, uxtb
 ; CHECK-GI-NEXT:    ld1.b { v1 }[7], [x8]
 ; CHECK-GI-NEXT:    fmov d0, d1
 ; CHECK-GI-NEXT:    add sp, sp, #64
@@ -1241,62 +1093,47 @@ define <8 x i8> @no_shuffle_bad_mask_index(<8 x i8> %src, <8 x i8> %mask) {
 ; CHECK-GI-NEXT:    sub sp, sp, #64
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-GI-NEXT:    movi.8b v2, #7
+; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    str d0, [sp]
 ; CHECK-GI-NEXT:    and.8b v2, v1, v2
 ; CHECK-GI-NEXT:    umov.b w8, v2[1]
-; CHECK-GI-NEXT:    umov.b w10, v2[1]
-; CHECK-GI-NEXT:    lsl x9, x8, #1
-; CHECK-GI-NEXT:    sub x8, x9, x8
-; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    ldr b1, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[2]
+; CHECK-GI-NEXT:    umov.b w8, v2[1]
+; CHECK-GI-NEXT:    add x9, sp, #8
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
 ; CHECK-GI-NEXT:    mov.b v1[0], v1[0]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #8
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[3]
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[2]
+; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    str d0, [sp, #16]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
 ; CHECK-GI-NEXT:    mov.b v1[1], v3[0]
-; CHECK-GI-NEXT:    sub x8, x8, x9
-; CHECK-GI-NEXT:    add x9, sp, #16
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[4]
+; CHECK-GI-NEXT:    umov.b w8, v2[3]
+; CHECK-GI-NEXT:    add x9, sp, #24
 ; CHECK-GI-NEXT:    str d0, [sp, #24]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #24
 ; CHECK-GI-NEXT:    mov.b v1[2], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[5]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str d0, [sp, #32]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[4]
 ; CHECK-GI-NEXT:    add x9, sp, #32
+; CHECK-GI-NEXT:    str d0, [sp, #32]
 ; CHECK-GI-NEXT:    mov.b v1[3], v3[0]
 ; CHECK-GI-NEXT:    ldr b3, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
-; CHECK-GI-NEXT:    umov.b w9, v2[6]
+; CHECK-GI-NEXT:    umov.b w8, v2[5]
+; CHECK-GI-NEXT:    add x9, sp, #40
 ; CHECK-GI-NEXT:    str d0, [sp, #40]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x10, sp, #40
 ; CHECK-GI-NEXT:    mov.b v1[4], v3[0]
-; CHECK-GI-NEXT:    ldr b3, [x10, x8]
-; CHECK-GI-NEXT:    umov.b w10, v2[7]
-; CHECK-GI-NEXT:    lsl x8, x9, #1
-; CHECK-GI-NEXT:    str d0, [sp, #48]
-; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    umov.b w8, v2[6]
+; CHECK-GI-NEXT:    mov b2, v2[7]
 ; CHECK-GI-NEXT:    add x9, sp, #48
+; CHECK-GI-NEXT:    str d0, [sp, #48]
 ; CHECK-GI-NEXT:    mov.b v1[5], v3[0]
-; CHECK-GI-NEXT:    ldr b2, [x9, x8]
-; CHECK-GI-NEXT:    lsl x8, x10, #1
+; CHECK-GI-NEXT:    ldr b3, [x9, x8]
+; CHECK-GI-NEXT:    fmov w8, s2
 ; CHECK-GI-NEXT:    add x9, sp, #56
 ; CHECK-GI-NEXT:    str d0, [sp, #56]
-; CHECK-GI-NEXT:    sub x8, x8, x10
-; CHECK-GI-NEXT:    add x8, x9, x8
-; CHECK-GI-NEXT:    mov.b v1[6], v2[0]
+; CHECK-GI-NEXT:    mov.b v1[6], v3[0]
+; CHECK-GI-NEXT:    add x8, x9, w8, uxtb
 ; CHECK-GI-NEXT:    ld1.b { v1 }[7], [x8]
 ; CHECK-GI-NEXT:    fmov d0, d1
 ; CHECK-GI-NEXT:    add sp, sp, #64

>From 7d91302a3eefcd1f1c425261ec6d253332e78a42 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Mon, 28 Sep 2026 15:05:38 +0530
Subject: [PATCH 2/2] Added a postlegalizercombiner-mul-identity MIR test case.

---
 .../postlegalizercombiner-mul-identity.mir    | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/postlegalizercombiner-mul-identity.mir

diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizercombiner-mul-identity.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizercombiner-mul-identity.mir
new file mode 100644
index 0000000000000..ea4ea72ecb121
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizercombiner-mul-identity.mir
@@ -0,0 +1,50 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner %s -o - | FileCheck %s
+
+# The generic 'right_identity_one_int' combine matches (G_MUL x, 1) and is more specific 
+# than AArch64's `mul_const` combine, so it must win. mul_const still rewrites other 
+# constants scenarios like:
+# (x * 2^3 - 1) becomes (x << 3) - x.
+---
+name:            mul_by_one
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: mul_by_one
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %x:_(i64) = COPY $x0
+    ; CHECK-NEXT: $x0 = COPY %x(i64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %x:_(i64) = COPY $x0
+    %one:_(i64) = G_CONSTANT i64 1
+    %mul:_(i64) = G_MUL %x, %one
+    $x0 = COPY %mul(i64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            mul_by_seven
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: mul_by_seven
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %x:_(i64) = COPY $x0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i64) = G_SHL %x, [[C]](i64)
+    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i64) = G_SUB [[SHL]], %x
+    ; CHECK-NEXT: $x0 = COPY [[SUB]](i64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %x:_(i64) = COPY $x0
+    %seven:_(i64) = G_CONSTANT i64 7
+    %mul:_(i64) = G_MUL %x, %seven
+    $x0 = COPY %mul(i64)
+    RET_ReallyLR implicit $x0



More information about the llvm-commits mailing list