[llvm] [GlobalISel] [AArch64] Generate NEG for shift by (N - X) in SelectShiftMask (PR #226114)
Deepak Shirke via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 03:58:13 PDT 2026
https://github.com/deepakshirkem created https://github.com/llvm/llvm-project/pull/226114
AArch64 shift instructions only use the low log2(ShiftWidth) bits of the shift amount. When shifting by N-X where N == 0 mod ShiftWidth, this is equivalent to shifting by -X, so we can generate a NEG instead of a SUB from a constant.
For example, shl x, (sub 64, y) becomes shl x, neg(y).
This implements both SelectionDAG and GlobalISel. The GlobalISel side builds the NEG (SUBWrr/SUBXrr with the zero register) in the ComplexPattern renderer and constrains its operands with constrainSelectedInstRegOperands.
Part of the incremental work tracked in #224245 to replace tryShiftAmountMod with ComplexPattern-based isel.
Depends on #225842.
CC: @davemgreen @RKSimon @arsenm @topperc
>From 5a690e803ee696727b50c798fc0576e4d1e2a60c Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Mon, 14 Sep 2026 18:14:17 +0530
Subject: [PATCH 1/3] AArch64: Skip redundant shift amount masking using isel
ComplexPattern
AArch64 shift instructions (LSL/LSR/ASR) only use the low 5 bits (i32)
or 6 bits (i64) of the shift amount. Add SelectShiftMask ComplexPattern
and GIComplexOperandMatcher that strip redundant AND masks or zero-extensions
from shift amounts during instruction selection, when the mask exactly
covers a narrow type (byte, halfword, or word).
This mirrors what AArch64DAGToDAGISel::tryShiftAmountMod does, and adds
equivalent GlobalISel support via GIComplexPatternEquiv.
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 21 +++
.../lib/Target/AArch64/AArch64InstrFormats.td | 15 +++
.../GISel/AArch64InstructionSelector.cpp | 49 +++++++
...st-and-by-const-from-lshr-in-eqcmp-zero.ll | 47 +++----
...ist-and-by-const-from-shl-in-eqcmp-zero.ll | 24 ++--
llvm/test/CodeGen/AArch64/select_const.ll | 3 -
llvm/test/CodeGen/AArch64/shift-mod.ll | 124 +++++++++++++++---
llvm/test/CodeGen/AArch64/shift.ll | 96 ++++----------
8 files changed, 245 insertions(+), 134 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index feb8f465837940..9d5722e1961734 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -89,6 +89,9 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
return SelectShiftedRegister(N, true, Reg, Shift);
}
+ template <unsigned ShiftWidth>
+ bool SelectShiftMask(SDValue N, SDValue &ShAmt);
+
bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
return SelectAddrModeIndexed7S(N, 1, Base, OffImm);
}
@@ -766,6 +769,24 @@ bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
return true;
}
+template <unsigned ShiftWidth>
+bool AArch64DAGToDAGISel::SelectShiftMask(SDValue N, SDValue &ShAmt) {
+ // AArch64 shift instructions only use the low log2(ShiftWidth) bits of the
+ // shift amount. If the shift amount has a redundant AND mask that covers
+ // those bits, we can remove it. Return false if nothing was combined so
+ // other patterns (e.g. zext/sext GPR32 → SUBREG_TO_REG) can match.
+ if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1)) &&
+ N.getValueType() == (ShiftWidth == 32 ? MVT::i32 : MVT::i64)) {
+ uint64_t Mask = N.getConstantOperandVal(1);
+ // Remove AND if the mask covers the low log2(ShiftWidth) bits.
+ if ((Mask & (ShiftWidth - 1)) == (ShiftWidth - 1)) {
+ ShAmt = N.getOperand(0);
+ return true;
+ }
+ }
+ return false;
+}
+
/// SelectArithImmed - Select an immediate value that can be represented as
/// a 12-bit value shifted left by either 0 or 12. If so, return true with
/// Val set to the 12-bit value and Shift set to the shifter operand.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index ea5164c9569928..ebac6e48739739 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -2749,6 +2749,16 @@ class BaseShift<bit size, bits<2> shift_type, RegisterClass regtype, string asm,
let Inst{11-10} = shift_type;
}
+// ComplexPattern to match shift amounts with redundant AND masks.
+// AArch64 shift instructions only use the low 5 bits (i32) or 6 bits (i64),
+// so AND masks that exactly cover a narrow type are redundant.
+def shiftMask32 : ComplexPattern<i32, 1, "SelectShiftMask<32>", [], [], 0>;
+def gi_shift_mask32 : GIComplexOperandMatcher<s32, "selectShiftMask32">,
+ GIComplexPatternEquiv<shiftMask32>;
+def shiftMask64 : ComplexPattern<i64, 1, "SelectShiftMask<64>", [], [], 0>;
+def gi_shift_mask64 : GIComplexOperandMatcher<s64, "selectShiftMask64">,
+ GIComplexPatternEquiv<shiftMask64>;
+
multiclass Shift<bits<2> shift_type, string asm, SDNode OpNode> {
def Wr : BaseShift<0b0, shift_type, GPR32, asm>;
@@ -2774,6 +2784,11 @@ multiclass Shift<bits<2> shift_type, string asm, SDNode OpNode> {
def : Pat<(i64 (OpNode GPR64:$Rn, (i64 (zext GPR32:$Rm)))),
(!cast<Instruction>(NAME # "Xr") GPR64:$Rn,
(SUBREG_TO_REG GPR32:$Rm, sub_32))>;
+
+ def : Pat<(i32 (OpNode GPR32:$Rn, shiftMask32:$Rm)),
+ (!cast<Instruction>(NAME # "Wr") GPR32:$Rn, shiftMask32:$Rm)>;
+ def : Pat<(i64 (OpNode GPR64:$Rn, shiftMask64:$Rm)),
+ (!cast<Instruction>(NAME # "Xr") GPR64:$Rn, shiftMask64:$Rm)>;
}
class ShiftAlias<string asm, Instruction inst, RegisterClass regtype>
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index f42e1864d41b54..c740c2119d0f5e 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -384,6 +384,10 @@ class AArch64InstructionSelector : public InstructionSelector {
ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;
+ template <unsigned ShiftWidth>
+ ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
+ ComplexRendererFns selectShiftMask32(MachineOperand &Root) const;
+ ComplexRendererFns selectShiftMask64(MachineOperand &Root) const;
ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;
@@ -7301,6 +7305,51 @@ AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
}
+template <unsigned ShiftWidth>
+InstructionSelector::ComplexRendererFns
+AArch64InstructionSelector::selectShiftMask(MachineOperand &Root) const {
+ if (!Root.isReg())
+ return std::nullopt;
+
+ MachineRegisterInfo &MRI =
+ Root.getParent()->getParent()->getParent()->getRegInfo();
+
+ Register ShAmtReg = Root.getReg();
+
+ // Peek through zext for i32 shifts only. For i64 shifts the zext case
+ // is already handled by existing patterns in the Shift multiclass.
+ if (ShiftWidth == 32) {
+ Register ZExtSrcReg;
+ if (mi_match(ShAmtReg, MRI, m_GZExt(m_Reg(ZExtSrcReg))))
+ ShAmtReg = ZExtSrcReg;
+ }
+
+ // Remove redundant AND mask introduced by legalization of a narrow zext.
+ // Only strip masks that exactly cover a narrow type (byte, halfword, word)
+ // and where the AND result type matches the shift width, to avoid removing
+ // intentional masks used in fshl/fshr computations.
+ APInt AndMask;
+ Register AndSrcReg;
+ if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask))) &&
+ MRI.getType(ShAmtReg).getSizeInBits() == ShiftWidth) {
+ uint64_t UMask = AndMask.getZExtValue();
+ if (UMask == 0xff || UMask == 0xffff || UMask == 0xffffffff)
+ ShAmtReg = AndSrcReg;
+ }
+
+ return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
+}
+
+InstructionSelector::ComplexRendererFns
+AArch64InstructionSelector::selectShiftMask32(MachineOperand &Root) const {
+ return selectShiftMask<32>(Root);
+}
+
+InstructionSelector::ComplexRendererFns
+AArch64InstructionSelector::selectShiftMask64(MachineOperand &Root) const {
+ return selectShiftMask<64>(Root);
+}
+
/// Helper to select an immediate value that can be represented as a 12-bit
/// value shifted left by either 0 or 12. If it is possible to do so, return
/// the immediate and shift value. If not, return std::nullopt.
diff --git a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
index 7d8ff9ac11e307..78efedb0c93170 100644
--- a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
@@ -24,8 +24,7 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_signbit_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #128 // =0x80
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsr w8, w8, w9
+; CHECK-GI-NEXT: lsr w8, w8, w1
; CHECK-GI-NEXT: tst w8, w0
; CHECK-GI-NEXT: cset w0, eq
; CHECK-GI-NEXT: ret
@@ -46,8 +45,7 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_lowestbit_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsr w8, w8, w9
+; CHECK-GI-NEXT: lsr w8, w8, w1
; CHECK-GI-NEXT: tst w8, w0
; CHECK-GI-NEXT: cset w0, eq
; CHECK-GI-NEXT: ret
@@ -68,8 +66,7 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_bitsinmiddle_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #24 // =0x18
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsr w8, w8, w9
+; CHECK-GI-NEXT: lsr w8, w8, w1
; CHECK-GI-NEXT: tst w8, w0
; CHECK-GI-NEXT: cset w0, eq
; CHECK-GI-NEXT: ret
@@ -92,8 +89,7 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
; CHECK-GI-LABEL: scalar_i16_signbit_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #32768 // =0x8000
-; CHECK-GI-NEXT: and w9, w1, #0xffff
-; CHECK-GI-NEXT: lsr w8, w8, w9
+; CHECK-GI-NEXT: lsr w8, w8, w1
; CHECK-GI-NEXT: tst w8, w0
; CHECK-GI-NEXT: cset w0, eq
; CHECK-GI-NEXT: ret
@@ -114,8 +110,7 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
; CHECK-GI-LABEL: scalar_i16_lowestbit_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and w9, w1, #0xffff
-; CHECK-GI-NEXT: lsr w8, w8, w9
+; CHECK-GI-NEXT: lsr w8, w8, w1
; CHECK-GI-NEXT: tst w8, w0
; CHECK-GI-NEXT: cset w0, eq
; CHECK-GI-NEXT: ret
@@ -136,8 +131,7 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
; CHECK-GI-LABEL: scalar_i16_bitsinmiddle_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #4080 // =0xff0
-; CHECK-GI-NEXT: and w9, w1, #0xffff
-; CHECK-GI-NEXT: lsr w8, w8, w9
+; CHECK-GI-NEXT: lsr w8, w8, w1
; CHECK-GI-NEXT: tst w8, w0
; CHECK-GI-NEXT: cset w0, eq
; CHECK-GI-NEXT: ret
@@ -425,8 +419,7 @@ define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_signbit_ne:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #128 // =0x80
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsr w8, w8, w9
+; CHECK-GI-NEXT: lsr w8, w8, w1
; CHECK-GI-NEXT: tst w8, w0
; CHECK-GI-NEXT: cset w0, ne
; CHECK-GI-NEXT: ret
@@ -480,24 +473,14 @@ define i1 @scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
}
define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
-; CHECK-SD-LABEL: scalar_i8_signbit_eq_with_nonzero:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: mov w8, #128 // =0x80
-; CHECK-SD-NEXT: lsr w8, w8, w1
-; CHECK-SD-NEXT: and w8, w8, w0
-; CHECK-SD-NEXT: cmp w8, #1
-; CHECK-SD-NEXT: cset w0, eq
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: scalar_i8_signbit_eq_with_nonzero:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov w8, #128 // =0x80
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsr w8, w8, w9
-; CHECK-GI-NEXT: and w8, w8, w0
-; CHECK-GI-NEXT: cmp w8, #1
-; CHECK-GI-NEXT: cset w0, eq
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: scalar_i8_signbit_eq_with_nonzero:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov w8, #128 // =0x80
+; CHECK-NEXT: lsr w8, w8, w1
+; CHECK-NEXT: and w8, w8, w0
+; CHECK-NEXT: cmp w8, #1
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
%t0 = lshr i8 128, %y
%t1 = and i8 %t0, %x
%res = icmp eq i8 %t1, 1 ; should be comparing with 0
diff --git a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
index f61e4303ca2be8..b55efeb580ee73 100644
--- a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
@@ -25,8 +25,7 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_signbit_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #-128 // =0xffffff80
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsl w8, w8, w9
+; CHECK-GI-NEXT: lsl w8, w8, w1
; CHECK-GI-NEXT: and w8, w8, w0
; CHECK-GI-NEXT: tst w8, #0xff
; CHECK-GI-NEXT: cset w0, eq
@@ -49,8 +48,7 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_lowestbit_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsl w8, w8, w9
+; CHECK-GI-NEXT: lsl w8, w8, w1
; CHECK-GI-NEXT: and w8, w8, w0
; CHECK-GI-NEXT: tst w8, #0xff
; CHECK-GI-NEXT: cset w0, eq
@@ -73,8 +71,7 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_bitsinmiddle_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #24 // =0x18
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsl w8, w8, w9
+; CHECK-GI-NEXT: lsl w8, w8, w1
; CHECK-GI-NEXT: and w8, w8, w0
; CHECK-GI-NEXT: tst w8, #0xff
; CHECK-GI-NEXT: cset w0, eq
@@ -99,8 +96,7 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
; CHECK-GI-LABEL: scalar_i16_signbit_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #-32768 // =0xffff8000
-; CHECK-GI-NEXT: and w9, w1, #0xffff
-; CHECK-GI-NEXT: lsl w8, w8, w9
+; CHECK-GI-NEXT: lsl w8, w8, w1
; CHECK-GI-NEXT: and w8, w8, w0
; CHECK-GI-NEXT: tst w8, #0xffff
; CHECK-GI-NEXT: cset w0, eq
@@ -123,8 +119,7 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
; CHECK-GI-LABEL: scalar_i16_lowestbit_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and w9, w1, #0xffff
-; CHECK-GI-NEXT: lsl w8, w8, w9
+; CHECK-GI-NEXT: lsl w8, w8, w1
; CHECK-GI-NEXT: and w8, w8, w0
; CHECK-GI-NEXT: tst w8, #0xffff
; CHECK-GI-NEXT: cset w0, eq
@@ -147,8 +142,7 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
; CHECK-GI-LABEL: scalar_i16_bitsinmiddle_eq:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #4080 // =0xff0
-; CHECK-GI-NEXT: and w9, w1, #0xffff
-; CHECK-GI-NEXT: lsl w8, w8, w9
+; CHECK-GI-NEXT: lsl w8, w8, w1
; CHECK-GI-NEXT: and w8, w8, w0
; CHECK-GI-NEXT: tst w8, #0xffff
; CHECK-GI-NEXT: cset w0, eq
@@ -430,8 +424,7 @@ define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_signbit_ne:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #-128 // =0xffffff80
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsl w8, w8, w9
+; CHECK-GI-NEXT: lsl w8, w8, w1
; CHECK-GI-NEXT: and w8, w8, w0
; CHECK-GI-NEXT: tst w8, #0xff
; CHECK-GI-NEXT: cset w0, ne
@@ -488,8 +481,7 @@ define i1 @scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
; CHECK-GI-LABEL: scalar_i8_bitsinmiddle_slt:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #24 // =0x18
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: lsl w8, w8, w9
+; CHECK-GI-NEXT: lsl w8, w8, w1
; CHECK-GI-NEXT: and w8, w8, w0
; CHECK-GI-NEXT: sxtb w8, w8
; CHECK-GI-NEXT: cmp w8, #0
diff --git a/llvm/test/CodeGen/AArch64/select_const.ll b/llvm/test/CodeGen/AArch64/select_const.ll
index daa9971dbcd58f..c67d527dbfed11 100644
--- a/llvm/test/CodeGen/AArch64/select_const.ll
+++ b/llvm/test/CodeGen/AArch64/select_const.ll
@@ -672,7 +672,6 @@ define i8 @shl_constant_sel_constants(i1 %cond) {
; CHECK-GI-NEXT: sbfx w9, w0, #0, #1
; CHECK-GI-NEXT: mov w8, #1 // =0x1
; CHECK-GI-NEXT: add w9, w9, #3
-; CHECK-GI-NEXT: and w9, w9, #0xff
; CHECK-GI-NEXT: lsl w0, w8, w9
; CHECK-GI-NEXT: ret
%sel = select i1 %cond, i8 2, i8 3
@@ -714,7 +713,6 @@ define i8 @lshr_constant_sel_constants(i1 %cond) {
; CHECK-GI-NEXT: sbfx w9, w0, #0, #1
; CHECK-GI-NEXT: mov w8, #64 // =0x40
; CHECK-GI-NEXT: add w9, w9, #3
-; CHECK-GI-NEXT: and w9, w9, #0xff
; CHECK-GI-NEXT: lsr w0, w8, w9
; CHECK-GI-NEXT: ret
%sel = select i1 %cond, i8 2, i8 3
@@ -747,7 +745,6 @@ define i8 @ashr_constant_sel_constants(i1 %cond) {
; CHECK-GI-NEXT: sbfx w9, w0, #0, #1
; CHECK-GI-NEXT: mov w8, #-128 // =0xffffff80
; CHECK-GI-NEXT: add w9, w9, #3
-; CHECK-GI-NEXT: and w9, w9, #0xff
; CHECK-GI-NEXT: asr w0, w8, w9
; CHECK-GI-NEXT: ret
%sel = select i1 %cond, i8 2, i8 3
diff --git a/llvm/test/CodeGen/AArch64/shift-mod.ll b/llvm/test/CodeGen/AArch64/shift-mod.ll
index a235b3628ded5e..c9333fb757a79a 100644
--- a/llvm/test/CodeGen/AArch64/shift-mod.ll
+++ b/llvm/test/CodeGen/AArch64/shift-mod.ll
@@ -52,11 +52,17 @@ define i64 @test3(i64 %x, i64 %y) {
}
define i64 @test4(i64 %y, i32 %s) {
-; CHECK-LABEL: test4:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT: asr x0, x0, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test4:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT: asr x0, x0, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test4:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov w8, w1
+; CHECK-GI-NEXT: asr x0, x0, x8
+; CHECK-GI-NEXT: ret
entry:
%sh_prom = zext i32 %s to i64
%shr = ashr i64 %y, %sh_prom
@@ -64,11 +70,18 @@ entry:
}
define i64 @test5(i64 %y, i32 %s) {
-; CHECK-LABEL: test5:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT: asr x0, x0, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test5:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT: asr x0, x0, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test5:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT: sxtw x8, w1
+; CHECK-GI-NEXT: asr x0, x0, x8
+; CHECK-GI-NEXT: ret
entry:
%sh_prom = sext i32 %s to i64
%shr = ashr i64 %y, %sh_prom
@@ -76,11 +89,18 @@ entry:
}
define i64 @test6(i64 %y, i32 %s) {
-; CHECK-LABEL: test6:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT: lsl x0, x0, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test6:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT: lsl x0, x0, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test6:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT: sxtw x8, w1
+; CHECK-GI-NEXT: lsl x0, x0, x8
+; CHECK-GI-NEXT: ret
entry:
%sh_prom = sext i32 %s to i64
%shr = shl i64 %y, %sh_prom
@@ -191,3 +211,77 @@ define i64 @ashr_add_shl_mismatch_shifts2(i64 %r) {
%conv1 = ashr i64 %sext, 8
ret i64 %conv1
}
+
+; Test that narrow shift amounts (i8/i16) don't generate redundant AND.
+define i32 @shl_i32_i8(i32 %x, i8 %amt) {
+; CHECK-LABEL: shl_i32_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl w0, w0, w1
+; CHECK-NEXT: ret
+ %ext = zext i8 %amt to i32
+ %r = shl i32 %x, %ext
+ ret i32 %r
+}
+
+define i32 @lshr_i32_i8(i32 %x, i8 %amt) {
+; CHECK-LABEL: lshr_i32_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsr w0, w0, w1
+; CHECK-NEXT: ret
+ %ext = zext i8 %amt to i32
+ %r = lshr i32 %x, %ext
+ ret i32 %r
+}
+
+define i32 @ashr_i32_i8(i32 %x, i8 %amt) {
+; CHECK-LABEL: ashr_i32_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: asr w0, w0, w1
+; CHECK-NEXT: ret
+ %ext = zext i8 %amt to i32
+ %r = ashr i32 %x, %ext
+ ret i32 %r
+}
+
+define i32 @shl_i32_i16(i32 %x, i16 %amt) {
+; CHECK-LABEL: shl_i32_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl w0, w0, w1
+; CHECK-NEXT: ret
+ %ext = zext i16 %amt to i32
+ %r = shl i32 %x, %ext
+ ret i32 %r
+}
+
+define i64 @shl_i64_i8(i64 %x, i8 %amt) {
+; CHECK-LABEL: shl_i64_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT: lsl x0, x0, x1
+; CHECK-NEXT: ret
+ %ext = zext i8 %amt to i64
+ %r = shl i64 %x, %ext
+ ret i64 %r
+}
+
+define i64 @lshr_i64_i8(i64 %x, i8 %amt) {
+; CHECK-LABEL: lshr_i64_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT: lsr x0, x0, x1
+; CHECK-NEXT: ret
+ %ext = zext i8 %amt to i64
+ %r = lshr i64 %x, %ext
+ ret i64 %r
+}
+
+define i64 @ashr_i64_i16(i64 %x, i16 %amt) {
+; CHECK-LABEL: ashr_i64_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT: asr x0, x0, x1
+; CHECK-NEXT: ret
+ %ext = zext i16 %amt to i64
+ %r = ashr i64 %x, %ext
+ ret i64 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/shift.ll b/llvm/test/CodeGen/AArch64/shift.ll
index 5d7935474c9035..eb5f56f771a0f2 100644
--- a/llvm/test/CodeGen/AArch64/shift.ll
+++ b/llvm/test/CodeGen/AArch64/shift.ll
@@ -19,31 +19,19 @@ define i1 @shl_i1(i1 %0, i1 %1){
}
define i8 @shl_i8(i8 %0, i8 %1){
-; CHECK-SD-LABEL: shl_i8:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: lsl w0, w0, w1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: shl_i8:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: and w8, w1, #0xff
-; CHECK-GI-NEXT: lsl w0, w0, w8
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: shl_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl w0, w0, w1
+; CHECK-NEXT: ret
%3 = shl i8 %0, %1
ret i8 %3
}
define i16 @shl_i16(i16 %0, i16 %1){
-; CHECK-SD-LABEL: shl_i16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: lsl w0, w0, w1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: shl_i16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: and w8, w1, #0xffff
-; CHECK-GI-NEXT: lsl w0, w0, w8
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: shl_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl w0, w0, w1
+; CHECK-NEXT: ret
%3 = shl i16 %0, %1
ret i16 %3
}
@@ -118,35 +106,21 @@ define i1 @ashr_i1(i1 %0, i1 %1){
}
define i8 @ashr_i8(i8 %0, i8 %1){
-; CHECK-SD-LABEL: ashr_i8:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sxtb w8, w0
-; CHECK-SD-NEXT: asr w0, w8, w1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: ashr_i8:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: sxtb w8, w0
-; CHECK-GI-NEXT: and w9, w1, #0xff
-; CHECK-GI-NEXT: asr w0, w8, w9
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: ashr_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sxtb w8, w0
+; CHECK-NEXT: asr w0, w8, w1
+; CHECK-NEXT: ret
%3 = ashr i8 %0, %1
ret i8 %3
}
define i16 @ashr_i16(i16 %0, i16 %1){
-; CHECK-SD-LABEL: ashr_i16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sxth w8, w0
-; CHECK-SD-NEXT: asr w0, w8, w1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: ashr_i16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: sxth w8, w0
-; CHECK-GI-NEXT: and w9, w1, #0xffff
-; CHECK-GI-NEXT: asr w0, w8, w9
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: ashr_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sxth w8, w0
+; CHECK-NEXT: asr w0, w8, w1
+; CHECK-NEXT: ret
%3 = ashr i16 %0, %1
ret i16 %3
}
@@ -223,35 +197,21 @@ define i1 @lshr_i1(i1 %0, i1 %1){
}
define i8 @lshr_i8(i8 %0, i8 %1){
-; CHECK-SD-LABEL: lshr_i8:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: and w8, w0, #0xff
-; CHECK-SD-NEXT: lsr w0, w8, w1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: lshr_i8:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: and w8, w1, #0xff
-; CHECK-GI-NEXT: and w9, w0, #0xff
-; CHECK-GI-NEXT: lsr w0, w9, w8
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: lshr_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: and w8, w0, #0xff
+; CHECK-NEXT: lsr w0, w8, w1
+; CHECK-NEXT: ret
%3 = lshr i8 %0, %1
ret i8 %3
}
define i16 @lshr_i16(i16 %0, i16 %1){
-; CHECK-SD-LABEL: lshr_i16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: and w8, w0, #0xffff
-; CHECK-SD-NEXT: lsr w0, w8, w1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: lshr_i16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: and w8, w1, #0xffff
-; CHECK-GI-NEXT: and w9, w0, #0xffff
-; CHECK-GI-NEXT: lsr w0, w9, w8
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: lshr_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: and w8, w0, #0xffff
+; CHECK-NEXT: lsr w0, w8, w1
+; CHECK-NEXT: ret
%3 = lshr i16 %0, %1
ret i16 %3
}
>From cee429985557e8b977e65efb10454a32792572f4 Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Wed, 23 Sep 2026 22:09:29 +0530
Subject: [PATCH 2/3] AArch64: Skip ADD/SUB by multiple of shift size in
SelectShiftMask
Extend SelectShiftMask ComplexPattern to also handle ADD/SUB where the
immediate operand is a multiple of the shift width. AArch64 shift
instructions only use the low log2(ShiftWidth) bits of the shift amount,
so shifting by X+N where N == 0 mod ShiftWidth is equivalent to shifting
by X alone.
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 14 +
.../GISel/AArch64InstructionSelector.cpp | 15 +
.../test/CodeGen/AArch64/and-mask-variable.ll | 14 +-
llvm/test/CodeGen/AArch64/fcvt-i256.ll | 696 +++++++++---------
llvm/test/CodeGen/AArch64/fsh.ll | 526 ++++++-------
llvm/test/CodeGen/AArch64/funnel-shift.ll | 32 +-
llvm/test/CodeGen/AArch64/shift-mod.ll | 45 +-
llvm/test/CodeGen/AArch64/shift.ll | 116 ++-
8 files changed, 702 insertions(+), 756 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 9d5722e1961734..1a991ffcbdeebd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -784,6 +784,20 @@ bool AArch64DAGToDAGISel::SelectShiftMask(SDValue N, SDValue &ShAmt) {
return true;
}
}
+ // If shifting by X+/-N where N == 0 mod ShiftWidth, then just shift by X
+ // to avoid the ADD/SUB. Only do this if the ADD/SUB has a single use, so
+ // we don't leave the ADD/SUB behind for other users.
+ if ((N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) &&
+ N.hasOneUse() &&
+ N.getValueType() == (ShiftWidth == 32 ? MVT::i32 : MVT::i64)) {
+ uint64_t Imm;
+ if (isIntImmediate(N.getOperand(1).getNode(), Imm) &&
+ (Imm % ShiftWidth == 0)) {
+ ShAmt = N.getOperand(0);
+ return true;
+ }
+ }
+
return false;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index c740c2119d0f5e..de66ab60a9abd7 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -7337,6 +7337,21 @@ AArch64InstructionSelector::selectShiftMask(MachineOperand &Root) const {
ShAmtReg = AndSrcReg;
}
+ // If shifting by X+/-N where N == 0 mod ShiftWidth, then just shift by X
+ // to avoid the ADD/SUB. Only do this if the ADD/SUB has a single use, so
+ // we don't leave the ADD/SUB behind for other users.
+ Register AddSrcReg;
+ int64_t AddImm;
+ if (MRI.hasOneUse(ShAmtReg) &&
+ (mi_match(ShAmtReg, MRI,
+ m_GAdd(m_Reg(AddSrcReg), m_ICstOrSplat(AddImm))) ||
+ mi_match(ShAmtReg, MRI,
+ m_GSub(m_Reg(AddSrcReg), m_ICstOrSplat(AddImm)))) &&
+ (AddImm % ShiftWidth == 0)) {
+ ShAmtReg = AddSrcReg;
+ return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
+ }
+
return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
}
diff --git a/llvm/test/CodeGen/AArch64/and-mask-variable.ll b/llvm/test/CodeGen/AArch64/and-mask-variable.ll
index f41cdc6dd241b2..ea5079770b2435 100644
--- a/llvm/test/CodeGen/AArch64/and-mask-variable.ll
+++ b/llvm/test/CodeGen/AArch64/and-mask-variable.ll
@@ -58,16 +58,14 @@ define i128 @mask_pair_128(i128 %x, i128 %y) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #64 // =0x40
; CHECK-GI-NEXT: mov x9, #-1 // =0xffffffffffffffff
-; CHECK-GI-NEXT: sub x10, x2, #64
-; CHECK-GI-NEXT: sub x8, x8, x2
-; CHECK-GI-NEXT: lsl x11, x9, x2
; CHECK-GI-NEXT: cmp x2, #64
+; CHECK-GI-NEXT: sub x8, x8, x2
+; CHECK-GI-NEXT: lsl x10, x9, x2
; CHECK-GI-NEXT: lsr x8, x9, x8
-; CHECK-GI-NEXT: lsl x9, x9, x10
-; CHECK-GI-NEXT: csel x10, x11, xzr, lo
-; CHECK-GI-NEXT: orr x8, x8, x11
-; CHECK-GI-NEXT: and x0, x10, x0
-; CHECK-GI-NEXT: csel x8, x8, x9, lo
+; CHECK-GI-NEXT: csel x9, x10, xzr, lo
+; CHECK-GI-NEXT: orr x8, x8, x10
+; CHECK-GI-NEXT: and x0, x9, x0
+; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x2, #0
; CHECK-GI-NEXT: csinv x8, x8, xzr, ne
; CHECK-GI-NEXT: and x1, x8, x1
diff --git a/llvm/test/CodeGen/AArch64/fcvt-i256.ll b/llvm/test/CodeGen/AArch64/fcvt-i256.ll
index d7d9d041875064..1e4f9b1d250b81 100644
--- a/llvm/test/CodeGen/AArch64/fcvt-i256.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt-i256.ll
@@ -698,81 +698,77 @@ define i256 @f32_to_s256(float %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB4_3: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub w11, w11, #150
-; CHECK-GI-NEXT: mov w13, #64 // =0x40
-; CHECK-GI-NEXT: mov w12, #128 // =0x80
-; CHECK-GI-NEXT: sub x14, x11, #64
-; CHECK-GI-NEXT: sub x15, x13, x11
+; CHECK-GI-NEXT: mov w12, #64 // =0x40
+; CHECK-GI-NEXT: mov w13, #128 // =0x80
+; CHECK-GI-NEXT: sub x15, x12, x11
; CHECK-GI-NEXT: lsl x16, x10, x11
+; CHECK-GI-NEXT: sub x13, x13, x11
; CHECK-GI-NEXT: lsr x15, x10, x15
-; CHECK-GI-NEXT: lsl x14, x10, x14
-; CHECK-GI-NEXT: sub x12, x12, x11
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x17, x11, #128
-; CHECK-GI-NEXT: lsr x18, x10, x12
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x14, x15, x14, lo
+; CHECK-GI-NEXT: sub x14, x11, #128
+; CHECK-GI-NEXT: lsr x17, x10, x13
+; CHECK-GI-NEXT: csel x18, x16, xzr, lo
+; CHECK-GI-NEXT: sub x12, x12, x14
+; CHECK-GI-NEXT: csel x15, x15, x16, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: sub x15, x17, #64
-; CHECK-GI-NEXT: csel x14, xzr, x14, eq
-; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: sub x13, x13, x17
-; CHECK-GI-NEXT: csel x18, x18, xzr, lo
-; CHECK-GI-NEXT: cmp x12, #0
-; CHECK-GI-NEXT: lsl x12, x10, x17
-; CHECK-GI-NEXT: lsr x13, x10, x13
-; CHECK-GI-NEXT: lsl x15, x10, x15
-; CHECK-GI-NEXT: csel x18, x10, x18, eq
-; CHECK-GI-NEXT: cmp x17, #64
-; CHECK-GI-NEXT: csel x12, x12, xzr, lo
-; CHECK-GI-NEXT: csel x10, x13, x15, lo
-; CHECK-GI-NEXT: cmp x17, #0
-; CHECK-GI-NEXT: csel x13, xzr, x10, eq
+; CHECK-GI-NEXT: lsl x16, x10, x14
+; CHECK-GI-NEXT: csel x15, xzr, x15, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: lsr x12, x10, x12
+; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel x13, x10, x17, eq
+; CHECK-GI-NEXT: cmp x14, #64
+; CHECK-GI-NEXT: csel x17, x16, xzr, lo
+; CHECK-GI-NEXT: csel x10, x12, x16, lo
+; CHECK-GI-NEXT: cmp x14, #0
+; CHECK-GI-NEXT: csel x12, xzr, x10, eq
; CHECK-GI-NEXT: cmp x11, #128
-; CHECK-GI-NEXT: csel x10, x16, xzr, lo
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x12, x18, x12, lo
-; CHECK-GI-NEXT: csel x13, xzr, x13, lo
+; CHECK-GI-NEXT: csel x10, x18, xzr, lo
+; CHECK-GI-NEXT: csel x15, x15, xzr, lo
+; CHECK-GI-NEXT: csel x13, x13, x17, lo
+; CHECK-GI-NEXT: csel x12, xzr, x12, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: umulh x11, x10, x9
-; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: umulh x18, x10, x9
; CHECK-GI-NEXT: csel x13, xzr, x13, eq
-; CHECK-GI-NEXT: umulh x18, x14, x9
-; CHECK-GI-NEXT: umulh x17, x12, x8
-; CHECK-GI-NEXT: umulh x15, x10, x8
-; CHECK-GI-NEXT: add x18, x18, x11
+; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: umulh x11, x15, x9
+; CHECK-GI-NEXT: umulh x17, x13, x8
+; CHECK-GI-NEXT: umulh x14, x10, x8
+; CHECK-GI-NEXT: add x11, x11, x18
; CHECK-GI-NEXT: mul x16, x10, x9
-; CHECK-GI-NEXT: add x17, x17, x18
-; CHECK-GI-NEXT: mul x0, x14, x8
-; CHECK-GI-NEXT: madd x17, x10, x9, x17
-; CHECK-GI-NEXT: adds x15, x15, x16
-; CHECK-GI-NEXT: umulh x2, x14, x8
+; CHECK-GI-NEXT: add x11, x17, x11
+; CHECK-GI-NEXT: mul x0, x15, x8
+; CHECK-GI-NEXT: umulh x2, x15, x8
+; CHECK-GI-NEXT: adds x14, x14, x16
+; CHECK-GI-NEXT: madd x11, x10, x9, x11
; CHECK-GI-NEXT: cset w3, hs
-; CHECK-GI-NEXT: adds x1, x15, x0
-; CHECK-GI-NEXT: mul x18, x14, x9
-; CHECK-GI-NEXT: cset w15, hs
-; CHECK-GI-NEXT: madd x14, x14, x9, x17
-; CHECK-GI-NEXT: adds x11, x2, x11
-; CHECK-GI-NEXT: mul x17, x12, x8
+; CHECK-GI-NEXT: adds x1, x14, x0
+; CHECK-GI-NEXT: mul x17, x15, x9
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: adds x18, x2, x18
+; CHECK-GI-NEXT: madd x11, x15, x9, x11
; CHECK-GI-NEXT: cset w0, hs
-; CHECK-GI-NEXT: adds x11, x11, x16
-; CHECK-GI-NEXT: and x16, x0, #0x1
-; CHECK-GI-NEXT: madd x9, x12, x9, x14
-; CHECK-GI-NEXT: and x12, x3, #0x1
-; CHECK-GI-NEXT: and x14, x15, #0x1
-; CHECK-GI-NEXT: cset w15, hs
-; CHECK-GI-NEXT: adds x11, x11, x18
-; CHECK-GI-NEXT: and x15, x15, #0x1
+; CHECK-GI-NEXT: adds x16, x18, x16
+; CHECK-GI-NEXT: and x18, x0, #0x1
+; CHECK-GI-NEXT: mul x15, x13, x8
+; CHECK-GI-NEXT: madd x9, x13, x9, x11
+; CHECK-GI-NEXT: and x11, x3, #0x1
+; CHECK-GI-NEXT: and x13, x14, #0x1
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: and x14, x14, #0x1
; CHECK-GI-NEXT: mul x0, x10, x8
-; CHECK-GI-NEXT: add x10, x12, x14
+; CHECK-GI-NEXT: add x10, x11, x13
+; CHECK-GI-NEXT: adds x13, x16, x17
+; CHECK-GI-NEXT: add x11, x18, x14
; CHECK-GI-NEXT: cset w14, hs
-; CHECK-GI-NEXT: adds x11, x11, x17
-; CHECK-GI-NEXT: add x12, x16, x15
-; CHECK-GI-NEXT: madd x8, x13, x8, x9
+; CHECK-GI-NEXT: adds x13, x13, x15
+; CHECK-GI-NEXT: madd x8, x12, x8, x9
; CHECK-GI-NEXT: and x9, x14, #0x1
-; CHECK-GI-NEXT: cset w13, hs
-; CHECK-GI-NEXT: adds x2, x11, x10
-; CHECK-GI-NEXT: add x9, x12, x9
-; CHECK-GI-NEXT: and x10, x13, #0x1
+; CHECK-GI-NEXT: cset w12, hs
+; CHECK-GI-NEXT: adds x2, x13, x10
+; CHECK-GI-NEXT: add x9, x11, x9
+; CHECK-GI-NEXT: and x10, x12, #0x1
; CHECK-GI-NEXT: cset w11, hs
; CHECK-GI-NEXT: add x9, x9, x10
; CHECK-GI-NEXT: and x10, x11, #0x1
@@ -871,40 +867,36 @@ define i256 @f32_to_u256(float %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB5_3: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub w9, w9, #150
-; CHECK-GI-NEXT: mov w11, #64 // =0x40
-; CHECK-GI-NEXT: mov w10, #128 // =0x80
-; CHECK-GI-NEXT: sub x12, x9, #64
-; CHECK-GI-NEXT: sub x13, x11, x9
-; CHECK-GI-NEXT: lsl x14, x8, x9
-; CHECK-GI-NEXT: lsr x13, x8, x13
-; CHECK-GI-NEXT: lsl x12, x8, x12
-; CHECK-GI-NEXT: sub x10, x10, x9
+; CHECK-GI-NEXT: mov w10, #64 // =0x40
+; CHECK-GI-NEXT: mov w11, #128 // =0x80
+; CHECK-GI-NEXT: sub x12, x10, x9
+; CHECK-GI-NEXT: lsl x13, x8, x9
+; CHECK-GI-NEXT: sub x11, x11, x9
+; CHECK-GI-NEXT: lsr x12, x8, x12
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: sub x15, x9, #128
-; CHECK-GI-NEXT: lsr x16, x8, x10
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x12, x13, x12, lo
+; CHECK-GI-NEXT: lsr x14, x8, x11
+; CHECK-GI-NEXT: csel x15, x13, xzr, lo
+; CHECK-GI-NEXT: csel x12, x12, x13, lo
; CHECK-GI-NEXT: cmp x9, #0
-; CHECK-GI-NEXT: sub x13, x15, #64
+; CHECK-GI-NEXT: sub x13, x9, #128
; CHECK-GI-NEXT: csel x12, xzr, x12, eq
-; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: sub x11, x11, x15
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: cmp x10, #0
-; CHECK-GI-NEXT: lsl x10, x8, x15
-; CHECK-GI-NEXT: lsr x11, x8, x11
-; CHECK-GI-NEXT: lsl x13, x8, x13
-; CHECK-GI-NEXT: csel x8, x8, x16, eq
-; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: csel x10, x10, xzr, lo
-; CHECK-GI-NEXT: csel x11, x11, x13, lo
-; CHECK-GI-NEXT: cmp x15, #0
-; CHECK-GI-NEXT: csel x11, xzr, x11, eq
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: sub x10, x10, x13
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: cmp x11, #0
+; CHECK-GI-NEXT: lsl x11, x8, x13
+; CHECK-GI-NEXT: lsr x10, x8, x10
+; CHECK-GI-NEXT: csel x8, x8, x14, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x14, x11, xzr, lo
+; CHECK-GI-NEXT: csel x10, x10, x11, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel x10, xzr, x10, eq
; CHECK-GI-NEXT: cmp x9, #128
-; CHECK-GI-NEXT: csel x0, x14, xzr, lo
+; CHECK-GI-NEXT: csel x0, x15, xzr, lo
; CHECK-GI-NEXT: csel x1, x12, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x10, lo
-; CHECK-GI-NEXT: csel x10, xzr, x11, lo
+; CHECK-GI-NEXT: csel x8, x8, x14, lo
+; CHECK-GI-NEXT: csel x10, xzr, x10, lo
; CHECK-GI-NEXT: cmp x9, #0
; CHECK-GI-NEXT: csel x2, xzr, x8, eq
; CHECK-GI-NEXT: csel x3, xzr, x10, eq
@@ -1059,81 +1051,77 @@ define i256 @f64_to_s256(double %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB6_3: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub x11, x11, #1075
-; CHECK-GI-NEXT: mov w13, #64 // =0x40
-; CHECK-GI-NEXT: mov w12, #128 // =0x80
-; CHECK-GI-NEXT: sub x14, x11, #64
-; CHECK-GI-NEXT: sub x15, x13, x11
+; CHECK-GI-NEXT: mov w12, #64 // =0x40
+; CHECK-GI-NEXT: mov w13, #128 // =0x80
+; CHECK-GI-NEXT: sub x15, x12, x11
; CHECK-GI-NEXT: lsl x16, x10, x11
+; CHECK-GI-NEXT: sub x13, x13, x11
; CHECK-GI-NEXT: lsr x15, x10, x15
-; CHECK-GI-NEXT: lsl x14, x10, x14
-; CHECK-GI-NEXT: sub x12, x12, x11
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x17, x11, #128
-; CHECK-GI-NEXT: lsr x18, x10, x12
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x14, x15, x14, lo
+; CHECK-GI-NEXT: sub x14, x11, #128
+; CHECK-GI-NEXT: lsr x17, x10, x13
+; CHECK-GI-NEXT: csel x18, x16, xzr, lo
+; CHECK-GI-NEXT: sub x12, x12, x14
+; CHECK-GI-NEXT: csel x15, x15, x16, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: sub x15, x17, #64
-; CHECK-GI-NEXT: csel x14, xzr, x14, eq
-; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: sub x13, x13, x17
-; CHECK-GI-NEXT: csel x18, x18, xzr, lo
-; CHECK-GI-NEXT: cmp x12, #0
-; CHECK-GI-NEXT: lsl x12, x10, x17
-; CHECK-GI-NEXT: lsr x13, x10, x13
-; CHECK-GI-NEXT: lsl x15, x10, x15
-; CHECK-GI-NEXT: csel x18, x10, x18, eq
-; CHECK-GI-NEXT: cmp x17, #64
-; CHECK-GI-NEXT: csel x12, x12, xzr, lo
-; CHECK-GI-NEXT: csel x10, x13, x15, lo
-; CHECK-GI-NEXT: cmp x17, #0
-; CHECK-GI-NEXT: csel x13, xzr, x10, eq
+; CHECK-GI-NEXT: lsl x16, x10, x14
+; CHECK-GI-NEXT: csel x15, xzr, x15, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: lsr x12, x10, x12
+; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel x13, x10, x17, eq
+; CHECK-GI-NEXT: cmp x14, #64
+; CHECK-GI-NEXT: csel x17, x16, xzr, lo
+; CHECK-GI-NEXT: csel x10, x12, x16, lo
+; CHECK-GI-NEXT: cmp x14, #0
+; CHECK-GI-NEXT: csel x12, xzr, x10, eq
; CHECK-GI-NEXT: cmp x11, #128
-; CHECK-GI-NEXT: csel x10, x16, xzr, lo
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x12, x18, x12, lo
-; CHECK-GI-NEXT: csel x13, xzr, x13, lo
+; CHECK-GI-NEXT: csel x10, x18, xzr, lo
+; CHECK-GI-NEXT: csel x15, x15, xzr, lo
+; CHECK-GI-NEXT: csel x13, x13, x17, lo
+; CHECK-GI-NEXT: csel x12, xzr, x12, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: umulh x11, x10, x9
-; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: umulh x18, x10, x9
; CHECK-GI-NEXT: csel x13, xzr, x13, eq
-; CHECK-GI-NEXT: umulh x18, x14, x9
-; CHECK-GI-NEXT: umulh x17, x12, x8
-; CHECK-GI-NEXT: umulh x15, x10, x8
-; CHECK-GI-NEXT: add x18, x18, x11
+; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: umulh x11, x15, x9
+; CHECK-GI-NEXT: umulh x17, x13, x8
+; CHECK-GI-NEXT: umulh x14, x10, x8
+; CHECK-GI-NEXT: add x11, x11, x18
; CHECK-GI-NEXT: mul x16, x10, x9
-; CHECK-GI-NEXT: add x17, x17, x18
-; CHECK-GI-NEXT: mul x0, x14, x8
-; CHECK-GI-NEXT: madd x17, x10, x9, x17
-; CHECK-GI-NEXT: adds x15, x15, x16
-; CHECK-GI-NEXT: umulh x2, x14, x8
+; CHECK-GI-NEXT: add x11, x17, x11
+; CHECK-GI-NEXT: mul x0, x15, x8
+; CHECK-GI-NEXT: umulh x2, x15, x8
+; CHECK-GI-NEXT: adds x14, x14, x16
+; CHECK-GI-NEXT: madd x11, x10, x9, x11
; CHECK-GI-NEXT: cset w3, hs
-; CHECK-GI-NEXT: adds x1, x15, x0
-; CHECK-GI-NEXT: mul x18, x14, x9
-; CHECK-GI-NEXT: cset w15, hs
-; CHECK-GI-NEXT: madd x14, x14, x9, x17
-; CHECK-GI-NEXT: adds x11, x2, x11
-; CHECK-GI-NEXT: mul x17, x12, x8
+; CHECK-GI-NEXT: adds x1, x14, x0
+; CHECK-GI-NEXT: mul x17, x15, x9
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: adds x18, x2, x18
+; CHECK-GI-NEXT: madd x11, x15, x9, x11
; CHECK-GI-NEXT: cset w0, hs
-; CHECK-GI-NEXT: adds x11, x11, x16
-; CHECK-GI-NEXT: and x16, x0, #0x1
-; CHECK-GI-NEXT: madd x9, x12, x9, x14
-; CHECK-GI-NEXT: and x12, x3, #0x1
-; CHECK-GI-NEXT: and x14, x15, #0x1
-; CHECK-GI-NEXT: cset w15, hs
-; CHECK-GI-NEXT: adds x11, x11, x18
-; CHECK-GI-NEXT: and x15, x15, #0x1
+; CHECK-GI-NEXT: adds x16, x18, x16
+; CHECK-GI-NEXT: and x18, x0, #0x1
+; CHECK-GI-NEXT: mul x15, x13, x8
+; CHECK-GI-NEXT: madd x9, x13, x9, x11
+; CHECK-GI-NEXT: and x11, x3, #0x1
+; CHECK-GI-NEXT: and x13, x14, #0x1
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: and x14, x14, #0x1
; CHECK-GI-NEXT: mul x0, x10, x8
-; CHECK-GI-NEXT: add x10, x12, x14
+; CHECK-GI-NEXT: add x10, x11, x13
+; CHECK-GI-NEXT: adds x13, x16, x17
+; CHECK-GI-NEXT: add x11, x18, x14
; CHECK-GI-NEXT: cset w14, hs
-; CHECK-GI-NEXT: adds x11, x11, x17
-; CHECK-GI-NEXT: add x12, x16, x15
-; CHECK-GI-NEXT: madd x8, x13, x8, x9
+; CHECK-GI-NEXT: adds x13, x13, x15
+; CHECK-GI-NEXT: madd x8, x12, x8, x9
; CHECK-GI-NEXT: and x9, x14, #0x1
-; CHECK-GI-NEXT: cset w13, hs
-; CHECK-GI-NEXT: adds x2, x11, x10
-; CHECK-GI-NEXT: add x9, x12, x9
-; CHECK-GI-NEXT: and x10, x13, #0x1
+; CHECK-GI-NEXT: cset w12, hs
+; CHECK-GI-NEXT: adds x2, x13, x10
+; CHECK-GI-NEXT: add x9, x11, x9
+; CHECK-GI-NEXT: and x10, x12, #0x1
; CHECK-GI-NEXT: cset w11, hs
; CHECK-GI-NEXT: add x9, x9, x10
; CHECK-GI-NEXT: and x10, x11, #0x1
@@ -1232,40 +1220,36 @@ define i256 @f64_to_u256(double %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB7_3: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub x9, x9, #1075
-; CHECK-GI-NEXT: mov w11, #64 // =0x40
-; CHECK-GI-NEXT: mov w10, #128 // =0x80
-; CHECK-GI-NEXT: sub x12, x9, #64
-; CHECK-GI-NEXT: sub x13, x11, x9
-; CHECK-GI-NEXT: lsl x14, x8, x9
-; CHECK-GI-NEXT: lsr x13, x8, x13
-; CHECK-GI-NEXT: lsl x12, x8, x12
-; CHECK-GI-NEXT: sub x10, x10, x9
+; CHECK-GI-NEXT: mov w10, #64 // =0x40
+; CHECK-GI-NEXT: mov w11, #128 // =0x80
+; CHECK-GI-NEXT: sub x12, x10, x9
+; CHECK-GI-NEXT: lsl x13, x8, x9
+; CHECK-GI-NEXT: sub x11, x11, x9
+; CHECK-GI-NEXT: lsr x12, x8, x12
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: sub x15, x9, #128
-; CHECK-GI-NEXT: lsr x16, x8, x10
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x12, x13, x12, lo
+; CHECK-GI-NEXT: lsr x14, x8, x11
+; CHECK-GI-NEXT: csel x15, x13, xzr, lo
+; CHECK-GI-NEXT: csel x12, x12, x13, lo
; CHECK-GI-NEXT: cmp x9, #0
-; CHECK-GI-NEXT: sub x13, x15, #64
+; CHECK-GI-NEXT: sub x13, x9, #128
; CHECK-GI-NEXT: csel x12, xzr, x12, eq
-; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: sub x11, x11, x15
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: cmp x10, #0
-; CHECK-GI-NEXT: lsl x10, x8, x15
-; CHECK-GI-NEXT: lsr x11, x8, x11
-; CHECK-GI-NEXT: lsl x13, x8, x13
-; CHECK-GI-NEXT: csel x8, x8, x16, eq
-; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: csel x10, x10, xzr, lo
-; CHECK-GI-NEXT: csel x11, x11, x13, lo
-; CHECK-GI-NEXT: cmp x15, #0
-; CHECK-GI-NEXT: csel x11, xzr, x11, eq
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: sub x10, x10, x13
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: cmp x11, #0
+; CHECK-GI-NEXT: lsl x11, x8, x13
+; CHECK-GI-NEXT: lsr x10, x8, x10
+; CHECK-GI-NEXT: csel x8, x8, x14, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x14, x11, xzr, lo
+; CHECK-GI-NEXT: csel x10, x10, x11, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel x10, xzr, x10, eq
; CHECK-GI-NEXT: cmp x9, #128
-; CHECK-GI-NEXT: csel x0, x14, xzr, lo
+; CHECK-GI-NEXT: csel x0, x15, xzr, lo
; CHECK-GI-NEXT: csel x1, x12, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x10, lo
-; CHECK-GI-NEXT: csel x10, xzr, x11, lo
+; CHECK-GI-NEXT: csel x8, x8, x14, lo
+; CHECK-GI-NEXT: csel x10, xzr, x10, lo
; CHECK-GI-NEXT: cmp x9, #0
; CHECK-GI-NEXT: csel x2, xzr, x8, eq
; CHECK-GI-NEXT: csel x3, xzr, x10, eq
@@ -1448,81 +1432,77 @@ define i256 @f32_to_s256_sat(float %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB8_6: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub w11, w11, #150
-; CHECK-GI-NEXT: mov w13, #64 // =0x40
-; CHECK-GI-NEXT: mov w12, #128 // =0x80
-; CHECK-GI-NEXT: sub x14, x11, #64
-; CHECK-GI-NEXT: sub x15, x13, x11
+; CHECK-GI-NEXT: mov w12, #64 // =0x40
+; CHECK-GI-NEXT: mov w13, #128 // =0x80
+; CHECK-GI-NEXT: sub x15, x12, x11
; CHECK-GI-NEXT: lsl x16, x10, x11
+; CHECK-GI-NEXT: sub x13, x13, x11
; CHECK-GI-NEXT: lsr x15, x10, x15
-; CHECK-GI-NEXT: lsl x14, x10, x14
-; CHECK-GI-NEXT: sub x12, x12, x11
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x17, x11, #128
-; CHECK-GI-NEXT: lsr x18, x10, x12
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x14, x15, x14, lo
+; CHECK-GI-NEXT: sub x14, x11, #128
+; CHECK-GI-NEXT: lsr x17, x10, x13
+; CHECK-GI-NEXT: csel x18, x16, xzr, lo
+; CHECK-GI-NEXT: sub x12, x12, x14
+; CHECK-GI-NEXT: csel x15, x15, x16, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: sub x15, x17, #64
-; CHECK-GI-NEXT: csel x14, xzr, x14, eq
-; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: sub x13, x13, x17
-; CHECK-GI-NEXT: csel x18, x18, xzr, lo
-; CHECK-GI-NEXT: cmp x12, #0
-; CHECK-GI-NEXT: lsl x12, x10, x17
-; CHECK-GI-NEXT: lsr x13, x10, x13
-; CHECK-GI-NEXT: lsl x15, x10, x15
-; CHECK-GI-NEXT: csel x18, x10, x18, eq
-; CHECK-GI-NEXT: cmp x17, #64
-; CHECK-GI-NEXT: csel x12, x12, xzr, lo
-; CHECK-GI-NEXT: csel x10, x13, x15, lo
-; CHECK-GI-NEXT: cmp x17, #0
-; CHECK-GI-NEXT: csel x13, xzr, x10, eq
+; CHECK-GI-NEXT: lsl x16, x10, x14
+; CHECK-GI-NEXT: csel x15, xzr, x15, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: lsr x12, x10, x12
+; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel x13, x10, x17, eq
+; CHECK-GI-NEXT: cmp x14, #64
+; CHECK-GI-NEXT: csel x17, x16, xzr, lo
+; CHECK-GI-NEXT: csel x10, x12, x16, lo
+; CHECK-GI-NEXT: cmp x14, #0
+; CHECK-GI-NEXT: csel x12, xzr, x10, eq
; CHECK-GI-NEXT: cmp x11, #128
-; CHECK-GI-NEXT: csel x10, x16, xzr, lo
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x12, x18, x12, lo
-; CHECK-GI-NEXT: csel x13, xzr, x13, lo
+; CHECK-GI-NEXT: csel x10, x18, xzr, lo
+; CHECK-GI-NEXT: csel x15, x15, xzr, lo
+; CHECK-GI-NEXT: csel x13, x13, x17, lo
+; CHECK-GI-NEXT: csel x12, xzr, x12, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: umulh x11, x10, x9
-; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: umulh x18, x10, x9
; CHECK-GI-NEXT: csel x13, xzr, x13, eq
-; CHECK-GI-NEXT: umulh x18, x14, x9
-; CHECK-GI-NEXT: umulh x17, x12, x8
-; CHECK-GI-NEXT: umulh x15, x10, x8
-; CHECK-GI-NEXT: add x18, x18, x11
+; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: umulh x11, x15, x9
+; CHECK-GI-NEXT: umulh x17, x13, x8
+; CHECK-GI-NEXT: umulh x14, x10, x8
+; CHECK-GI-NEXT: add x11, x11, x18
; CHECK-GI-NEXT: mul x16, x10, x9
-; CHECK-GI-NEXT: add x17, x17, x18
-; CHECK-GI-NEXT: mul x0, x14, x8
-; CHECK-GI-NEXT: madd x17, x10, x9, x17
-; CHECK-GI-NEXT: adds x15, x15, x16
-; CHECK-GI-NEXT: umulh x2, x14, x8
+; CHECK-GI-NEXT: add x11, x17, x11
+; CHECK-GI-NEXT: mul x0, x15, x8
+; CHECK-GI-NEXT: umulh x2, x15, x8
+; CHECK-GI-NEXT: adds x14, x14, x16
+; CHECK-GI-NEXT: madd x11, x10, x9, x11
; CHECK-GI-NEXT: cset w3, hs
-; CHECK-GI-NEXT: adds x1, x15, x0
-; CHECK-GI-NEXT: mul x18, x14, x9
-; CHECK-GI-NEXT: cset w15, hs
-; CHECK-GI-NEXT: madd x14, x14, x9, x17
-; CHECK-GI-NEXT: adds x11, x2, x11
-; CHECK-GI-NEXT: mul x17, x12, x8
+; CHECK-GI-NEXT: adds x1, x14, x0
+; CHECK-GI-NEXT: mul x17, x15, x9
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: adds x18, x2, x18
+; CHECK-GI-NEXT: madd x11, x15, x9, x11
; CHECK-GI-NEXT: cset w0, hs
-; CHECK-GI-NEXT: adds x11, x11, x16
-; CHECK-GI-NEXT: and x16, x0, #0x1
-; CHECK-GI-NEXT: madd x9, x12, x9, x14
-; CHECK-GI-NEXT: and x12, x3, #0x1
-; CHECK-GI-NEXT: and x14, x15, #0x1
-; CHECK-GI-NEXT: cset w15, hs
-; CHECK-GI-NEXT: adds x11, x11, x18
-; CHECK-GI-NEXT: and x15, x15, #0x1
+; CHECK-GI-NEXT: adds x16, x18, x16
+; CHECK-GI-NEXT: and x18, x0, #0x1
+; CHECK-GI-NEXT: mul x15, x13, x8
+; CHECK-GI-NEXT: madd x9, x13, x9, x11
+; CHECK-GI-NEXT: and x11, x3, #0x1
+; CHECK-GI-NEXT: and x13, x14, #0x1
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: and x14, x14, #0x1
; CHECK-GI-NEXT: mul x0, x10, x8
-; CHECK-GI-NEXT: add x10, x12, x14
+; CHECK-GI-NEXT: add x10, x11, x13
+; CHECK-GI-NEXT: adds x13, x16, x17
+; CHECK-GI-NEXT: add x11, x18, x14
; CHECK-GI-NEXT: cset w14, hs
-; CHECK-GI-NEXT: adds x11, x11, x17
-; CHECK-GI-NEXT: add x12, x16, x15
-; CHECK-GI-NEXT: madd x8, x13, x8, x9
+; CHECK-GI-NEXT: adds x13, x13, x15
+; CHECK-GI-NEXT: madd x8, x12, x8, x9
; CHECK-GI-NEXT: and x9, x14, #0x1
-; CHECK-GI-NEXT: cset w13, hs
-; CHECK-GI-NEXT: adds x2, x11, x10
-; CHECK-GI-NEXT: add x9, x12, x9
-; CHECK-GI-NEXT: and x10, x13, #0x1
+; CHECK-GI-NEXT: cset w12, hs
+; CHECK-GI-NEXT: adds x2, x13, x10
+; CHECK-GI-NEXT: add x9, x11, x9
+; CHECK-GI-NEXT: and x10, x12, #0x1
; CHECK-GI-NEXT: cset w11, hs
; CHECK-GI-NEXT: add x9, x9, x10
; CHECK-GI-NEXT: and x10, x11, #0x1
@@ -1657,40 +1637,36 @@ define i256 @f32_to_u256_sat(float %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB9_7: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub w9, w9, #150
-; CHECK-GI-NEXT: mov w11, #64 // =0x40
-; CHECK-GI-NEXT: mov w10, #128 // =0x80
-; CHECK-GI-NEXT: sub x12, x9, #64
-; CHECK-GI-NEXT: sub x13, x11, x9
-; CHECK-GI-NEXT: lsl x14, x8, x9
-; CHECK-GI-NEXT: lsr x13, x8, x13
-; CHECK-GI-NEXT: lsl x12, x8, x12
-; CHECK-GI-NEXT: sub x10, x10, x9
+; CHECK-GI-NEXT: mov w10, #64 // =0x40
+; CHECK-GI-NEXT: mov w11, #128 // =0x80
+; CHECK-GI-NEXT: sub x12, x10, x9
+; CHECK-GI-NEXT: lsl x13, x8, x9
+; CHECK-GI-NEXT: sub x11, x11, x9
+; CHECK-GI-NEXT: lsr x12, x8, x12
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: sub x15, x9, #128
-; CHECK-GI-NEXT: lsr x16, x8, x10
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x12, x13, x12, lo
+; CHECK-GI-NEXT: lsr x14, x8, x11
+; CHECK-GI-NEXT: csel x15, x13, xzr, lo
+; CHECK-GI-NEXT: csel x12, x12, x13, lo
; CHECK-GI-NEXT: cmp x9, #0
-; CHECK-GI-NEXT: sub x13, x15, #64
+; CHECK-GI-NEXT: sub x13, x9, #128
; CHECK-GI-NEXT: csel x12, xzr, x12, eq
-; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: sub x11, x11, x15
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: cmp x10, #0
-; CHECK-GI-NEXT: lsl x10, x8, x15
-; CHECK-GI-NEXT: lsr x11, x8, x11
-; CHECK-GI-NEXT: lsl x13, x8, x13
-; CHECK-GI-NEXT: csel x8, x8, x16, eq
-; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: csel x10, x10, xzr, lo
-; CHECK-GI-NEXT: csel x11, x11, x13, lo
-; CHECK-GI-NEXT: cmp x15, #0
-; CHECK-GI-NEXT: csel x11, xzr, x11, eq
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: sub x10, x10, x13
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: cmp x11, #0
+; CHECK-GI-NEXT: lsl x11, x8, x13
+; CHECK-GI-NEXT: lsr x10, x8, x10
+; CHECK-GI-NEXT: csel x8, x8, x14, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x14, x11, xzr, lo
+; CHECK-GI-NEXT: csel x10, x10, x11, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel x10, xzr, x10, eq
; CHECK-GI-NEXT: cmp x9, #128
-; CHECK-GI-NEXT: csel x0, x14, xzr, lo
+; CHECK-GI-NEXT: csel x0, x15, xzr, lo
; CHECK-GI-NEXT: csel x1, x12, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x10, lo
-; CHECK-GI-NEXT: csel x10, xzr, x11, lo
+; CHECK-GI-NEXT: csel x8, x8, x14, lo
+; CHECK-GI-NEXT: csel x10, xzr, x10, lo
; CHECK-GI-NEXT: cmp x9, #0
; CHECK-GI-NEXT: csel x2, xzr, x8, eq
; CHECK-GI-NEXT: csel x3, xzr, x10, eq
@@ -1875,81 +1851,77 @@ define i256 @f64_to_s256_sat(double %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB10_6: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub x11, x11, #1075
-; CHECK-GI-NEXT: mov w13, #64 // =0x40
-; CHECK-GI-NEXT: mov w12, #128 // =0x80
-; CHECK-GI-NEXT: sub x14, x11, #64
-; CHECK-GI-NEXT: sub x15, x13, x11
+; CHECK-GI-NEXT: mov w12, #64 // =0x40
+; CHECK-GI-NEXT: mov w13, #128 // =0x80
+; CHECK-GI-NEXT: sub x15, x12, x11
; CHECK-GI-NEXT: lsl x16, x10, x11
+; CHECK-GI-NEXT: sub x13, x13, x11
; CHECK-GI-NEXT: lsr x15, x10, x15
-; CHECK-GI-NEXT: lsl x14, x10, x14
-; CHECK-GI-NEXT: sub x12, x12, x11
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x17, x11, #128
-; CHECK-GI-NEXT: lsr x18, x10, x12
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x14, x15, x14, lo
+; CHECK-GI-NEXT: sub x14, x11, #128
+; CHECK-GI-NEXT: lsr x17, x10, x13
+; CHECK-GI-NEXT: csel x18, x16, xzr, lo
+; CHECK-GI-NEXT: sub x12, x12, x14
+; CHECK-GI-NEXT: csel x15, x15, x16, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: sub x15, x17, #64
-; CHECK-GI-NEXT: csel x14, xzr, x14, eq
-; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: sub x13, x13, x17
-; CHECK-GI-NEXT: csel x18, x18, xzr, lo
-; CHECK-GI-NEXT: cmp x12, #0
-; CHECK-GI-NEXT: lsl x12, x10, x17
-; CHECK-GI-NEXT: lsr x13, x10, x13
-; CHECK-GI-NEXT: lsl x15, x10, x15
-; CHECK-GI-NEXT: csel x18, x10, x18, eq
-; CHECK-GI-NEXT: cmp x17, #64
-; CHECK-GI-NEXT: csel x12, x12, xzr, lo
-; CHECK-GI-NEXT: csel x10, x13, x15, lo
-; CHECK-GI-NEXT: cmp x17, #0
-; CHECK-GI-NEXT: csel x13, xzr, x10, eq
+; CHECK-GI-NEXT: lsl x16, x10, x14
+; CHECK-GI-NEXT: csel x15, xzr, x15, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: lsr x12, x10, x12
+; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel x13, x10, x17, eq
+; CHECK-GI-NEXT: cmp x14, #64
+; CHECK-GI-NEXT: csel x17, x16, xzr, lo
+; CHECK-GI-NEXT: csel x10, x12, x16, lo
+; CHECK-GI-NEXT: cmp x14, #0
+; CHECK-GI-NEXT: csel x12, xzr, x10, eq
; CHECK-GI-NEXT: cmp x11, #128
-; CHECK-GI-NEXT: csel x10, x16, xzr, lo
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x12, x18, x12, lo
-; CHECK-GI-NEXT: csel x13, xzr, x13, lo
+; CHECK-GI-NEXT: csel x10, x18, xzr, lo
+; CHECK-GI-NEXT: csel x15, x15, xzr, lo
+; CHECK-GI-NEXT: csel x13, x13, x17, lo
+; CHECK-GI-NEXT: csel x12, xzr, x12, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: umulh x11, x10, x9
-; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: umulh x18, x10, x9
; CHECK-GI-NEXT: csel x13, xzr, x13, eq
-; CHECK-GI-NEXT: umulh x18, x14, x9
-; CHECK-GI-NEXT: umulh x17, x12, x8
-; CHECK-GI-NEXT: umulh x15, x10, x8
-; CHECK-GI-NEXT: add x18, x18, x11
+; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: umulh x11, x15, x9
+; CHECK-GI-NEXT: umulh x17, x13, x8
+; CHECK-GI-NEXT: umulh x14, x10, x8
+; CHECK-GI-NEXT: add x11, x11, x18
; CHECK-GI-NEXT: mul x16, x10, x9
-; CHECK-GI-NEXT: add x17, x17, x18
-; CHECK-GI-NEXT: mul x0, x14, x8
-; CHECK-GI-NEXT: madd x17, x10, x9, x17
-; CHECK-GI-NEXT: adds x15, x15, x16
-; CHECK-GI-NEXT: umulh x2, x14, x8
+; CHECK-GI-NEXT: add x11, x17, x11
+; CHECK-GI-NEXT: mul x0, x15, x8
+; CHECK-GI-NEXT: umulh x2, x15, x8
+; CHECK-GI-NEXT: adds x14, x14, x16
+; CHECK-GI-NEXT: madd x11, x10, x9, x11
; CHECK-GI-NEXT: cset w3, hs
-; CHECK-GI-NEXT: adds x1, x15, x0
-; CHECK-GI-NEXT: mul x18, x14, x9
-; CHECK-GI-NEXT: cset w15, hs
-; CHECK-GI-NEXT: madd x14, x14, x9, x17
-; CHECK-GI-NEXT: adds x11, x2, x11
-; CHECK-GI-NEXT: mul x17, x12, x8
+; CHECK-GI-NEXT: adds x1, x14, x0
+; CHECK-GI-NEXT: mul x17, x15, x9
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: adds x18, x2, x18
+; CHECK-GI-NEXT: madd x11, x15, x9, x11
; CHECK-GI-NEXT: cset w0, hs
-; CHECK-GI-NEXT: adds x11, x11, x16
-; CHECK-GI-NEXT: and x16, x0, #0x1
-; CHECK-GI-NEXT: madd x9, x12, x9, x14
-; CHECK-GI-NEXT: and x12, x3, #0x1
-; CHECK-GI-NEXT: and x14, x15, #0x1
-; CHECK-GI-NEXT: cset w15, hs
-; CHECK-GI-NEXT: adds x11, x11, x18
-; CHECK-GI-NEXT: and x15, x15, #0x1
+; CHECK-GI-NEXT: adds x16, x18, x16
+; CHECK-GI-NEXT: and x18, x0, #0x1
+; CHECK-GI-NEXT: mul x15, x13, x8
+; CHECK-GI-NEXT: madd x9, x13, x9, x11
+; CHECK-GI-NEXT: and x11, x3, #0x1
+; CHECK-GI-NEXT: and x13, x14, #0x1
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: and x14, x14, #0x1
; CHECK-GI-NEXT: mul x0, x10, x8
-; CHECK-GI-NEXT: add x10, x12, x14
+; CHECK-GI-NEXT: add x10, x11, x13
+; CHECK-GI-NEXT: adds x13, x16, x17
+; CHECK-GI-NEXT: add x11, x18, x14
; CHECK-GI-NEXT: cset w14, hs
-; CHECK-GI-NEXT: adds x11, x11, x17
-; CHECK-GI-NEXT: add x12, x16, x15
-; CHECK-GI-NEXT: madd x8, x13, x8, x9
+; CHECK-GI-NEXT: adds x13, x13, x15
+; CHECK-GI-NEXT: madd x8, x12, x8, x9
; CHECK-GI-NEXT: and x9, x14, #0x1
-; CHECK-GI-NEXT: cset w13, hs
-; CHECK-GI-NEXT: adds x2, x11, x10
-; CHECK-GI-NEXT: add x9, x12, x9
-; CHECK-GI-NEXT: and x10, x13, #0x1
+; CHECK-GI-NEXT: cset w12, hs
+; CHECK-GI-NEXT: adds x2, x13, x10
+; CHECK-GI-NEXT: add x9, x11, x9
+; CHECK-GI-NEXT: and x10, x12, #0x1
; CHECK-GI-NEXT: cset w11, hs
; CHECK-GI-NEXT: add x9, x9, x10
; CHECK-GI-NEXT: and x10, x11, #0x1
@@ -2084,40 +2056,36 @@ define i256 @f64_to_u256_sat(double %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB11_7: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub x9, x9, #1075
-; CHECK-GI-NEXT: mov w11, #64 // =0x40
-; CHECK-GI-NEXT: mov w10, #128 // =0x80
-; CHECK-GI-NEXT: sub x12, x9, #64
-; CHECK-GI-NEXT: sub x13, x11, x9
-; CHECK-GI-NEXT: lsl x14, x8, x9
-; CHECK-GI-NEXT: lsr x13, x8, x13
-; CHECK-GI-NEXT: lsl x12, x8, x12
-; CHECK-GI-NEXT: sub x10, x10, x9
+; CHECK-GI-NEXT: mov w10, #64 // =0x40
+; CHECK-GI-NEXT: mov w11, #128 // =0x80
+; CHECK-GI-NEXT: sub x12, x10, x9
+; CHECK-GI-NEXT: lsl x13, x8, x9
+; CHECK-GI-NEXT: sub x11, x11, x9
+; CHECK-GI-NEXT: lsr x12, x8, x12
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: sub x15, x9, #128
-; CHECK-GI-NEXT: lsr x16, x8, x10
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x12, x13, x12, lo
+; CHECK-GI-NEXT: lsr x14, x8, x11
+; CHECK-GI-NEXT: csel x15, x13, xzr, lo
+; CHECK-GI-NEXT: csel x12, x12, x13, lo
; CHECK-GI-NEXT: cmp x9, #0
-; CHECK-GI-NEXT: sub x13, x15, #64
+; CHECK-GI-NEXT: sub x13, x9, #128
; CHECK-GI-NEXT: csel x12, xzr, x12, eq
-; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: sub x11, x11, x15
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: cmp x10, #0
-; CHECK-GI-NEXT: lsl x10, x8, x15
-; CHECK-GI-NEXT: lsr x11, x8, x11
-; CHECK-GI-NEXT: lsl x13, x8, x13
-; CHECK-GI-NEXT: csel x8, x8, x16, eq
-; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: csel x10, x10, xzr, lo
-; CHECK-GI-NEXT: csel x11, x11, x13, lo
-; CHECK-GI-NEXT: cmp x15, #0
-; CHECK-GI-NEXT: csel x11, xzr, x11, eq
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: sub x10, x10, x13
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: cmp x11, #0
+; CHECK-GI-NEXT: lsl x11, x8, x13
+; CHECK-GI-NEXT: lsr x10, x8, x10
+; CHECK-GI-NEXT: csel x8, x8, x14, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x14, x11, xzr, lo
+; CHECK-GI-NEXT: csel x10, x10, x11, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel x10, xzr, x10, eq
; CHECK-GI-NEXT: cmp x9, #128
-; CHECK-GI-NEXT: csel x0, x14, xzr, lo
+; CHECK-GI-NEXT: csel x0, x15, xzr, lo
; CHECK-GI-NEXT: csel x1, x12, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x10, lo
-; CHECK-GI-NEXT: csel x10, xzr, x11, lo
+; CHECK-GI-NEXT: csel x8, x8, x14, lo
+; CHECK-GI-NEXT: csel x10, xzr, x10, lo
; CHECK-GI-NEXT: cmp x9, #0
; CHECK-GI-NEXT: csel x2, xzr, x8, eq
; CHECK-GI-NEXT: csel x3, xzr, x10, eq
diff --git a/llvm/test/CodeGen/AArch64/fsh.ll b/llvm/test/CodeGen/AArch64/fsh.ll
index 312010904cf50c..84b88ae3da1399 100644
--- a/llvm/test/CodeGen/AArch64/fsh.ll
+++ b/llvm/test/CodeGen/AArch64/fsh.ll
@@ -181,35 +181,31 @@ define i128 @rotl_i128(i128 %a, i128 %c) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #64 // =0x40
; CHECK-GI-NEXT: and x9, x2, #0x7f
-; CHECK-GI-NEXT: neg x10, x2
-; CHECK-GI-NEXT: sub x12, x8, x9
-; CHECK-GI-NEXT: sub x11, x9, #64
-; CHECK-GI-NEXT: lsl x14, x1, x9
-; CHECK-GI-NEXT: lsr x12, x0, x12
+; CHECK-GI-NEXT: neg x11, x2
+; CHECK-GI-NEXT: sub x10, x8, x9
+; CHECK-GI-NEXT: lsl x12, x1, x9
+; CHECK-GI-NEXT: and x14, x11, #0x7f
+; CHECK-GI-NEXT: lsr x10, x0, x10
; CHECK-GI-NEXT: lsl x13, x0, x9
+; CHECK-GI-NEXT: sub x8, x8, x14
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: and x9, x10, #0x7f
-; CHECK-GI-NEXT: lsl x11, x0, x11
-; CHECK-GI-NEXT: orr x12, x12, x14
-; CHECK-GI-NEXT: sub x8, x8, x9
-; CHECK-GI-NEXT: sub x14, x9, #64
-; CHECK-GI-NEXT: csel x11, x12, x11, lo
-; CHECK-GI-NEXT: lsr x12, x0, x9
; CHECK-GI-NEXT: lsl x8, x1, x8
-; CHECK-GI-NEXT: csel x13, x13, xzr, lo
+; CHECK-GI-NEXT: orr x9, x10, x12
+; CHECK-GI-NEXT: lsr x10, x0, x14
+; CHECK-GI-NEXT: csel x12, x13, xzr, lo
+; CHECK-GI-NEXT: csel x9, x9, x13, lo
; CHECK-GI-NEXT: tst x2, #0x7f
-; CHECK-GI-NEXT: lsr x14, x1, x14
-; CHECK-GI-NEXT: csel x11, x1, x11, eq
-; CHECK-GI-NEXT: orr x8, x12, x8
-; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsr x12, x1, x9
-; CHECK-GI-NEXT: csel x8, x8, x14, lo
-; CHECK-GI-NEXT: tst x10, #0x7f
+; CHECK-GI-NEXT: lsr x13, x1, x14
+; CHECK-GI-NEXT: csel x9, x1, x9, eq
+; CHECK-GI-NEXT: orr x8, x10, x8
+; CHECK-GI-NEXT: cmp x14, #64
+; CHECK-GI-NEXT: csel x8, x8, x13, lo
+; CHECK-GI-NEXT: tst x11, #0x7f
; CHECK-GI-NEXT: csel x8, x0, x8, eq
-; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: csel x9, x12, xzr, lo
-; CHECK-GI-NEXT: orr x0, x13, x8
-; CHECK-GI-NEXT: orr x1, x11, x9
+; CHECK-GI-NEXT: cmp x14, #64
+; CHECK-GI-NEXT: csel x10, x13, xzr, lo
+; CHECK-GI-NEXT: orr x0, x12, x8
+; CHECK-GI-NEXT: orr x1, x9, x10
; CHECK-GI-NEXT: ret
entry:
%d = call i128 @llvm.fshl(i128 %a, i128 %a, i128 %c)
@@ -237,35 +233,31 @@ define i128 @rotr_i128(i128 %a, i128 %c) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #64 // =0x40
; CHECK-GI-NEXT: and x9, x2, #0x7f
-; CHECK-GI-NEXT: neg x13, x2
+; CHECK-GI-NEXT: neg x12, x2
; CHECK-GI-NEXT: sub x10, x8, x9
-; CHECK-GI-NEXT: sub x11, x9, #64
-; CHECK-GI-NEXT: lsr x12, x0, x9
+; CHECK-GI-NEXT: lsr x11, x0, x9
+; CHECK-GI-NEXT: lsr x13, x1, x9
; CHECK-GI-NEXT: lsl x10, x1, x10
-; CHECK-GI-NEXT: lsr x11, x1, x11
-; CHECK-GI-NEXT: and x14, x13, #0x7f
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: sub x8, x8, x14
-; CHECK-GI-NEXT: lsl x15, x1, x14
-; CHECK-GI-NEXT: orr x10, x12, x10
-; CHECK-GI-NEXT: lsr x12, x1, x9
-; CHECK-GI-NEXT: lsr x8, x0, x8
-; CHECK-GI-NEXT: csel x10, x10, x11, lo
-; CHECK-GI-NEXT: sub x11, x14, #64
+; CHECK-GI-NEXT: orr x10, x11, x10
+; CHECK-GI-NEXT: and x11, x12, #0x7f
+; CHECK-GI-NEXT: sub x8, x8, x11
+; CHECK-GI-NEXT: csel x10, x10, x13, lo
; CHECK-GI-NEXT: tst x2, #0x7f
+; CHECK-GI-NEXT: lsr x8, x0, x8
+; CHECK-GI-NEXT: lsl x14, x1, x11
; CHECK-GI-NEXT: csel x10, x0, x10, eq
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsl x9, x0, x14
-; CHECK-GI-NEXT: lsl x11, x0, x11
-; CHECK-GI-NEXT: csel x12, x12, xzr, lo
-; CHECK-GI-NEXT: orr x8, x8, x15
-; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: csel x9, x9, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x11, lo
-; CHECK-GI-NEXT: tst x13, #0x7f
+; CHECK-GI-NEXT: lsl x9, x0, x11
+; CHECK-GI-NEXT: csel x13, x13, xzr, lo
+; CHECK-GI-NEXT: orr x8, x8, x14
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: csel x11, x9, xzr, lo
+; CHECK-GI-NEXT: csel x8, x8, x9, lo
+; CHECK-GI-NEXT: tst x12, #0x7f
; CHECK-GI-NEXT: csel x8, x1, x8, eq
-; CHECK-GI-NEXT: orr x0, x10, x9
-; CHECK-GI-NEXT: orr x1, x12, x8
+; CHECK-GI-NEXT: orr x0, x10, x11
+; CHECK-GI-NEXT: orr x1, x13, x8
; CHECK-GI-NEXT: ret
entry:
%d = call i128 @llvm.fshr(i128 %a, i128 %a, i128 %c)
@@ -508,35 +500,31 @@ define i128 @fshl_i128(i128 %a, i128 %b, i128 %c) {
; CHECK-GI-NEXT: and x9, x4, #0x7f
; CHECK-GI-NEXT: mov w10, #127 // =0x7f
; CHECK-GI-NEXT: sub x12, x8, x9
-; CHECK-GI-NEXT: lsl x13, x1, x9
; CHECK-GI-NEXT: bic x10, x10, x4
+; CHECK-GI-NEXT: lsl x13, x1, x9
; CHECK-GI-NEXT: lsr x12, x0, x12
-; CHECK-GI-NEXT: sub x14, x9, #64
+; CHECK-GI-NEXT: extr x14, x3, x2, #1
+; CHECK-GI-NEXT: lsr x16, x3, #1
; CHECK-GI-NEXT: lsl x15, x0, x9
-; CHECK-GI-NEXT: extr x16, x3, x2, #1
-; CHECK-GI-NEXT: cmp x9, #64
; CHECK-GI-NEXT: sub x8, x8, x10
+; CHECK-GI-NEXT: cmp x9, #64
; CHECK-GI-NEXT: orr x9, x12, x13
-; CHECK-GI-NEXT: lsr x12, x3, #1
-; CHECK-GI-NEXT: lsl x13, x0, x14
-; CHECK-GI-NEXT: csel x14, x15, xzr, lo
-; CHECK-GI-NEXT: sub x15, x10, #64
-; CHECK-GI-NEXT: lsr x17, x16, x10
-; CHECK-GI-NEXT: lsl x8, x12, x8
-; CHECK-GI-NEXT: csel x9, x9, x13, lo
+; CHECK-GI-NEXT: lsr x13, x14, x10
+; CHECK-GI-NEXT: lsl x8, x16, x8
+; CHECK-GI-NEXT: csel x12, x15, xzr, lo
+; CHECK-GI-NEXT: csel x9, x9, x15, lo
; CHECK-GI-NEXT: tst x4, #0x7f
-; CHECK-GI-NEXT: lsr x13, x12, x15
+; CHECK-GI-NEXT: lsr x15, x16, x10
; CHECK-GI-NEXT: mvn x11, x4
; CHECK-GI-NEXT: csel x9, x1, x9, eq
-; CHECK-GI-NEXT: orr x8, x17, x8
+; CHECK-GI-NEXT: orr x8, x13, x8
; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: lsr x12, x12, x10
-; CHECK-GI-NEXT: csel x8, x8, x13, lo
+; CHECK-GI-NEXT: csel x8, x8, x15, lo
; CHECK-GI-NEXT: tst x11, #0x7f
-; CHECK-GI-NEXT: csel x8, x16, x8, eq
+; CHECK-GI-NEXT: csel x8, x14, x8, eq
; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: csel x10, x12, xzr, lo
-; CHECK-GI-NEXT: orr x0, x14, x8
+; CHECK-GI-NEXT: csel x10, x15, xzr, lo
+; CHECK-GI-NEXT: orr x0, x12, x8
; CHECK-GI-NEXT: orr x1, x9, x10
; CHECK-GI-NEXT: ret
entry:
@@ -572,32 +560,28 @@ define i128 @fshr_i128(i128 %a, i128 %b, i128 %c) {
; CHECK-GI-NEXT: and x14, x4, #0x7f
; CHECK-GI-NEXT: sub x12, x11, x8
; CHECK-GI-NEXT: lsl x13, x10, x8
-; CHECK-GI-NEXT: lsl x16, x9, x8
-; CHECK-GI-NEXT: lsr x12, x9, x12
-; CHECK-GI-NEXT: sub x17, x8, #64
-; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: lsl x8, x9, x17
; CHECK-GI-NEXT: sub x11, x11, x14
+; CHECK-GI-NEXT: lsr x12, x9, x12
+; CHECK-GI-NEXT: lsl x9, x9, x8
; CHECK-GI-NEXT: mvn x15, x4
-; CHECK-GI-NEXT: orr x12, x12, x13
-; CHECK-GI-NEXT: csel x9, x16, xzr, lo
-; CHECK-GI-NEXT: sub x13, x14, #64
-; CHECK-GI-NEXT: lsr x16, x2, x14
+; CHECK-GI-NEXT: cmp x8, #64
; CHECK-GI-NEXT: lsl x11, x3, x11
-; CHECK-GI-NEXT: csel x8, x12, x8, lo
+; CHECK-GI-NEXT: orr x8, x12, x13
+; CHECK-GI-NEXT: lsr x13, x2, x14
+; CHECK-GI-NEXT: csel x12, x9, xzr, lo
+; CHECK-GI-NEXT: csel x8, x8, x9, lo
; CHECK-GI-NEXT: tst x15, #0x7f
-; CHECK-GI-NEXT: lsr x12, x3, x13
+; CHECK-GI-NEXT: lsr x9, x3, x14
; CHECK-GI-NEXT: csel x8, x10, x8, eq
-; CHECK-GI-NEXT: orr x10, x16, x11
+; CHECK-GI-NEXT: orr x10, x13, x11
; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: lsr x11, x3, x14
-; CHECK-GI-NEXT: csel x10, x10, x12, lo
+; CHECK-GI-NEXT: csel x10, x10, x9, lo
; CHECK-GI-NEXT: tst x4, #0x7f
; CHECK-GI-NEXT: csel x10, x2, x10, eq
; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: csel x11, x11, xzr, lo
-; CHECK-GI-NEXT: orr x0, x9, x10
-; CHECK-GI-NEXT: orr x1, x8, x11
+; CHECK-GI-NEXT: csel x9, x9, xzr, lo
+; CHECK-GI-NEXT: orr x0, x12, x10
+; CHECK-GI-NEXT: orr x1, x8, x9
; CHECK-GI-NEXT: ret
entry:
%d = call i128 @llvm.fshr(i128 %a, i128 %b, i128 %c)
@@ -1762,65 +1746,57 @@ define <2 x i128> @rotl_v2i128(<2 x i128> %a, <2 x i128> %c) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: and x8, x4, #0x7f
; CHECK-GI-NEXT: mov w9, #64 // =0x40
-; CHECK-GI-NEXT: neg x13, x4
+; CHECK-GI-NEXT: and x11, x6, #0x7f
; CHECK-GI-NEXT: sub x10, x9, x8
; CHECK-GI-NEXT: lsl x12, x1, x8
-; CHECK-GI-NEXT: sub x11, x8, #64
+; CHECK-GI-NEXT: lsl x13, x0, x8
; CHECK-GI-NEXT: lsr x10, x0, x10
-; CHECK-GI-NEXT: lsl x14, x0, x8
-; CHECK-GI-NEXT: lsl x11, x0, x11
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: neg x15, x6
-; CHECK-GI-NEXT: orr x8, x10, x12
-; CHECK-GI-NEXT: and x10, x6, #0x7f
-; CHECK-GI-NEXT: csel x12, x14, xzr, lo
-; CHECK-GI-NEXT: sub x14, x9, x10
-; CHECK-GI-NEXT: csel x8, x8, x11, lo
+; CHECK-GI-NEXT: sub x8, x9, x11
+; CHECK-GI-NEXT: lsl x14, x3, x11
+; CHECK-GI-NEXT: lsr x8, x2, x8
+; CHECK-GI-NEXT: lsl x15, x2, x11
+; CHECK-GI-NEXT: orr x10, x10, x12
+; CHECK-GI-NEXT: csel x12, x13, xzr, lo
+; CHECK-GI-NEXT: neg x17, x6
+; CHECK-GI-NEXT: csel x10, x10, x13, lo
; CHECK-GI-NEXT: tst x4, #0x7f
-; CHECK-GI-NEXT: sub x11, x10, #64
-; CHECK-GI-NEXT: lsl x16, x2, x10
-; CHECK-GI-NEXT: lsr x14, x2, x14
-; CHECK-GI-NEXT: lsl x17, x3, x10
-; CHECK-GI-NEXT: csel x8, x1, x8, eq
-; CHECK-GI-NEXT: lsl x11, x2, x11
-; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: and x4, x15, #0x7f
-; CHECK-GI-NEXT: orr x10, x14, x17
-; CHECK-GI-NEXT: csel x14, x16, xzr, lo
-; CHECK-GI-NEXT: and x16, x13, #0x7f
-; CHECK-GI-NEXT: csel x10, x10, x11, lo
-; CHECK-GI-NEXT: sub x11, x9, x16
-; CHECK-GI-NEXT: sub x17, x16, #64
-; CHECK-GI-NEXT: lsr x18, x0, x16
-; CHECK-GI-NEXT: lsl x11, x1, x11
+; CHECK-GI-NEXT: neg x13, x4
+; CHECK-GI-NEXT: csel x10, x1, x10, eq
+; CHECK-GI-NEXT: orr x8, x8, x14
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: and x11, x13, #0x7f
+; CHECK-GI-NEXT: csel x14, x15, xzr, lo
+; CHECK-GI-NEXT: csel x8, x8, x15, lo
+; CHECK-GI-NEXT: sub x15, x9, x11
+; CHECK-GI-NEXT: lsr x16, x0, x11
; CHECK-GI-NEXT: tst x6, #0x7f
-; CHECK-GI-NEXT: lsr x17, x1, x17
-; CHECK-GI-NEXT: csel x10, x3, x10, eq
-; CHECK-GI-NEXT: cmp x16, #64
-; CHECK-GI-NEXT: orr x11, x18, x11
-; CHECK-GI-NEXT: sub x9, x9, x4
-; CHECK-GI-NEXT: lsr x1, x1, x16
-; CHECK-GI-NEXT: csel x11, x11, x17, lo
-; CHECK-GI-NEXT: tst x13, #0x7f
-; CHECK-GI-NEXT: sub x13, x4, #64
-; CHECK-GI-NEXT: lsr x17, x2, x4
+; CHECK-GI-NEXT: lsl x15, x1, x15
+; CHECK-GI-NEXT: lsr x18, x1, x11
+; CHECK-GI-NEXT: and x1, x17, #0x7f
+; CHECK-GI-NEXT: csel x8, x3, x8, eq
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: sub x9, x9, x1
+; CHECK-GI-NEXT: orr x15, x16, x15
; CHECK-GI-NEXT: lsl x9, x3, x9
-; CHECK-GI-NEXT: csel x11, x0, x11, eq
-; CHECK-GI-NEXT: cmp x16, #64
-; CHECK-GI-NEXT: lsr x13, x3, x13
-; CHECK-GI-NEXT: orr x0, x12, x11
-; CHECK-GI-NEXT: csel x16, x1, xzr, lo
-; CHECK-GI-NEXT: orr x9, x17, x9
-; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: lsr x17, x3, x4
-; CHECK-GI-NEXT: csel x9, x9, x13, lo
-; CHECK-GI-NEXT: tst x15, #0x7f
+; CHECK-GI-NEXT: csel x15, x15, x18, lo
+; CHECK-GI-NEXT: tst x13, #0x7f
+; CHECK-GI-NEXT: lsr x13, x2, x1
+; CHECK-GI-NEXT: csel x15, x0, x15, eq
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: lsr x11, x3, x1
+; CHECK-GI-NEXT: csel x16, x18, xzr, lo
+; CHECK-GI-NEXT: orr x9, x13, x9
+; CHECK-GI-NEXT: cmp x1, #64
+; CHECK-GI-NEXT: csel x9, x9, x11, lo
+; CHECK-GI-NEXT: tst x17, #0x7f
+; CHECK-GI-NEXT: orr x0, x12, x15
; CHECK-GI-NEXT: csel x9, x2, x9, eq
-; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: orr x1, x8, x16
-; CHECK-GI-NEXT: csel x13, x17, xzr, lo
+; CHECK-GI-NEXT: cmp x1, #64
+; CHECK-GI-NEXT: orr x1, x10, x16
+; CHECK-GI-NEXT: csel x11, x11, xzr, lo
; CHECK-GI-NEXT: orr x2, x14, x9
-; CHECK-GI-NEXT: orr x3, x10, x13
+; CHECK-GI-NEXT: orr x3, x8, x11
; CHECK-GI-NEXT: ret
entry:
%d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %a, <2 x i128> %c)
@@ -1860,65 +1836,57 @@ define <2 x i128> @rotr_v2i128(<2 x i128> %a, <2 x i128> %c) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: and x8, x4, #0x7f
; CHECK-GI-NEXT: mov w9, #64 // =0x40
-; CHECK-GI-NEXT: and x14, x6, #0x7f
+; CHECK-GI-NEXT: and x13, x6, #0x7f
; CHECK-GI-NEXT: sub x10, x9, x8
-; CHECK-GI-NEXT: sub x11, x8, #64
; CHECK-GI-NEXT: lsr x12, x0, x8
+; CHECK-GI-NEXT: lsr x14, x1, x8
; CHECK-GI-NEXT: lsl x10, x1, x10
-; CHECK-GI-NEXT: lsr x11, x1, x11
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: sub x15, x9, x14
-; CHECK-GI-NEXT: neg x13, x4
-; CHECK-GI-NEXT: lsr x17, x3, x14
+; CHECK-GI-NEXT: sub x15, x9, x13
+; CHECK-GI-NEXT: lsr x16, x2, x13
+; CHECK-GI-NEXT: neg x11, x4
; CHECK-GI-NEXT: orr x10, x12, x10
-; CHECK-GI-NEXT: lsr x12, x1, x8
-; CHECK-GI-NEXT: lsl x15, x3, x15
-; CHECK-GI-NEXT: csel x10, x10, x11, lo
+; CHECK-GI-NEXT: lsl x12, x3, x15
+; CHECK-GI-NEXT: lsr x15, x3, x13
+; CHECK-GI-NEXT: csel x10, x10, x14, lo
; CHECK-GI-NEXT: tst x4, #0x7f
-; CHECK-GI-NEXT: sub x11, x14, #64
; CHECK-GI-NEXT: csel x10, x0, x10, eq
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: lsr x8, x2, x14
-; CHECK-GI-NEXT: lsr x11, x3, x11
-; CHECK-GI-NEXT: csel x12, x12, xzr, lo
-; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: orr x8, x8, x15
-; CHECK-GI-NEXT: neg x16, x6
-; CHECK-GI-NEXT: csel x8, x8, x11, lo
+; CHECK-GI-NEXT: orr x8, x16, x12
+; CHECK-GI-NEXT: csel x12, x14, xzr, lo
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: and x14, x11, #0x7f
+; CHECK-GI-NEXT: csel x8, x8, x15, lo
; CHECK-GI-NEXT: tst x6, #0x7f
-; CHECK-GI-NEXT: and x11, x13, #0x7f
+; CHECK-GI-NEXT: neg x16, x6
+; CHECK-GI-NEXT: sub x17, x9, x14
; CHECK-GI-NEXT: csel x8, x2, x8, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: lsr x13, x0, x17
+; CHECK-GI-NEXT: lsl x17, x1, x14
+; CHECK-GI-NEXT: and x18, x16, #0x7f
+; CHECK-GI-NEXT: lsl x0, x0, x14
+; CHECK-GI-NEXT: sub x9, x9, x18
+; CHECK-GI-NEXT: csel x15, x15, xzr, lo
+; CHECK-GI-NEXT: orr x13, x13, x17
; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: sub x14, x9, x11
-; CHECK-GI-NEXT: sub x15, x11, #64
-; CHECK-GI-NEXT: lsr x14, x0, x14
-; CHECK-GI-NEXT: lsl x18, x1, x11
-; CHECK-GI-NEXT: lsl x4, x0, x11
-; CHECK-GI-NEXT: lsl x15, x0, x15
-; CHECK-GI-NEXT: and x0, x16, #0x7f
-; CHECK-GI-NEXT: csel x17, x17, xzr, lo
-; CHECK-GI-NEXT: orr x14, x14, x18
-; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x9, x9, x0
-; CHECK-GI-NEXT: csel x14, x14, x15, lo
-; CHECK-GI-NEXT: sub x15, x0, #64
; CHECK-GI-NEXT: lsr x9, x2, x9
-; CHECK-GI-NEXT: lsl x18, x3, x0
-; CHECK-GI-NEXT: csel x11, x4, xzr, lo
-; CHECK-GI-NEXT: tst x13, #0x7f
-; CHECK-GI-NEXT: lsl x13, x2, x0
-; CHECK-GI-NEXT: lsl x15, x2, x15
-; CHECK-GI-NEXT: csel x14, x1, x14, eq
-; CHECK-GI-NEXT: orr x9, x9, x18
-; CHECK-GI-NEXT: cmp x0, #64
-; CHECK-GI-NEXT: csel x13, x13, xzr, lo
-; CHECK-GI-NEXT: csel x9, x9, x15, lo
+; CHECK-GI-NEXT: lsl x17, x3, x18
+; CHECK-GI-NEXT: csel x14, x0, xzr, lo
+; CHECK-GI-NEXT: csel x13, x13, x0, lo
+; CHECK-GI-NEXT: tst x11, #0x7f
+; CHECK-GI-NEXT: lsl x11, x2, x18
+; CHECK-GI-NEXT: orr x0, x10, x14
+; CHECK-GI-NEXT: csel x13, x1, x13, eq
+; CHECK-GI-NEXT: orr x9, x9, x17
+; CHECK-GI-NEXT: cmp x18, #64
+; CHECK-GI-NEXT: csel x17, x11, xzr, lo
+; CHECK-GI-NEXT: csel x9, x9, x11, lo
; CHECK-GI-NEXT: tst x16, #0x7f
; CHECK-GI-NEXT: csel x9, x3, x9, eq
-; CHECK-GI-NEXT: orr x0, x10, x11
-; CHECK-GI-NEXT: orr x1, x12, x14
-; CHECK-GI-NEXT: orr x2, x8, x13
-; CHECK-GI-NEXT: orr x3, x17, x9
+; CHECK-GI-NEXT: orr x1, x12, x13
+; CHECK-GI-NEXT: orr x2, x8, x17
+; CHECK-GI-NEXT: orr x3, x15, x9
; CHECK-GI-NEXT: ret
entry:
%d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %a, <2 x i128> %c)
@@ -2862,78 +2830,66 @@ define <2 x i128> @fshl_v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
;
; CHECK-GI-LABEL: fshl_v2i128:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: str x19, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: .cfi_offset w19, -16
-; CHECK-GI-NEXT: ldr x11, [sp, #16]
-; CHECK-GI-NEXT: mov w9, #64 // =0x40
-; CHECK-GI-NEXT: ldr x12, [sp, #32]
-; CHECK-GI-NEXT: mov w13, #127 // =0x7f
-; CHECK-GI-NEXT: and x8, x11, #0x7f
-; CHECK-GI-NEXT: and x14, x12, #0x7f
-; CHECK-GI-NEXT: mvn x18, x11
-; CHECK-GI-NEXT: sub x10, x9, x8
-; CHECK-GI-NEXT: sub x15, x8, #64
-; CHECK-GI-NEXT: lsl x17, x1, x8
-; CHECK-GI-NEXT: lsr x16, x0, x10
-; CHECK-GI-NEXT: lsl x15, x0, x15
-; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: lsl x19, x0, x8
-; CHECK-GI-NEXT: lsl x0, x3, x14
-; CHECK-GI-NEXT: mvn x10, x12
-; CHECK-GI-NEXT: orr x16, x16, x17
-; CHECK-GI-NEXT: sub x17, x14, #64
-; CHECK-GI-NEXT: csel x15, x16, x15, lo
-; CHECK-GI-NEXT: sub x16, x9, x14
-; CHECK-GI-NEXT: csel x8, x19, xzr, lo
-; CHECK-GI-NEXT: lsr x16, x2, x16
-; CHECK-GI-NEXT: tst x11, #0x7f
-; CHECK-GI-NEXT: lsl x19, x2, x14
-; CHECK-GI-NEXT: lsl x17, x2, x17
-; CHECK-GI-NEXT: csel x15, x1, x15, eq
+; CHECK-GI-NEXT: ldr x8, [sp]
+; CHECK-GI-NEXT: mov w11, #64 // =0x40
+; CHECK-GI-NEXT: ldr x9, [sp, #16]
+; CHECK-GI-NEXT: mov w12, #127 // =0x7f
+; CHECK-GI-NEXT: and x10, x8, #0x7f
+; CHECK-GI-NEXT: and x14, x9, #0x7f
+; CHECK-GI-NEXT: mvn x16, x8
+; CHECK-GI-NEXT: sub x13, x11, x10
+; CHECK-GI-NEXT: lsl x15, x1, x10
+; CHECK-GI-NEXT: lsl x18, x0, x10
+; CHECK-GI-NEXT: lsr x13, x0, x13
+; CHECK-GI-NEXT: cmp x10, #64
+; CHECK-GI-NEXT: bic x17, x12, x8
+; CHECK-GI-NEXT: csel x0, x18, xzr, lo
+; CHECK-GI-NEXT: bic x12, x12, x9
+; CHECK-GI-NEXT: orr x13, x13, x15
+; CHECK-GI-NEXT: sub x15, x11, x14
+; CHECK-GI-NEXT: lsr x10, x2, x15
+; CHECK-GI-NEXT: lsl x15, x3, x14
+; CHECK-GI-NEXT: csel x13, x13, x18, lo
+; CHECK-GI-NEXT: tst x8, #0x7f
+; CHECK-GI-NEXT: csel x8, x1, x13, eq
+; CHECK-GI-NEXT: lsl x13, x2, x14
+; CHECK-GI-NEXT: orr x10, x10, x15
; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: orr x16, x16, x0
-; CHECK-GI-NEXT: bic x11, x13, x11
-; CHECK-GI-NEXT: csel x14, x19, xzr, lo
-; CHECK-GI-NEXT: csel x16, x16, x17, lo
-; CHECK-GI-NEXT: tst x12, #0x7f
-; CHECK-GI-NEXT: lsr x17, x5, #1
-; CHECK-GI-NEXT: extr x0, x5, x4, #1
-; CHECK-GI-NEXT: bic x12, x13, x12
-; CHECK-GI-NEXT: csel x13, x3, x16, eq
-; CHECK-GI-NEXT: sub x16, x9, x11
-; CHECK-GI-NEXT: sub x1, x11, #64
-; CHECK-GI-NEXT: lsr x3, x7, #1
-; CHECK-GI-NEXT: lsr x2, x0, x11
-; CHECK-GI-NEXT: lsl x16, x17, x16
-; CHECK-GI-NEXT: extr x4, x7, x6, #1
-; CHECK-GI-NEXT: lsr x1, x17, x1
-; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x9, x9, x12
-; CHECK-GI-NEXT: orr x16, x2, x16
-; CHECK-GI-NEXT: lsr x17, x17, x11
-; CHECK-GI-NEXT: lsl x9, x3, x9
-; CHECK-GI-NEXT: csel x16, x16, x1, lo
-; CHECK-GI-NEXT: tst x18, #0x7f
-; CHECK-GI-NEXT: sub x18, x12, #64
-; CHECK-GI-NEXT: lsr x1, x4, x12
-; CHECK-GI-NEXT: csel x16, x0, x16, eq
-; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: lsr x11, x3, x18
-; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: lsr x14, x5, #1
+; CHECK-GI-NEXT: extr x15, x5, x4, #1
+; CHECK-GI-NEXT: csel x18, x13, xzr, lo
+; CHECK-GI-NEXT: csel x10, x10, x13, lo
+; CHECK-GI-NEXT: sub x13, x11, x17
+; CHECK-GI-NEXT: tst x9, #0x7f
+; CHECK-GI-NEXT: lsr x1, x15, x17
+; CHECK-GI-NEXT: lsl x13, x14, x13
+; CHECK-GI-NEXT: csel x10, x3, x10, eq
+; CHECK-GI-NEXT: lsr x2, x7, #1
+; CHECK-GI-NEXT: lsr x14, x14, x17
+; CHECK-GI-NEXT: extr x3, x7, x6, #1
+; CHECK-GI-NEXT: orr x13, x1, x13
+; CHECK-GI-NEXT: cmp x17, #64
+; CHECK-GI-NEXT: sub x11, x11, x12
+; CHECK-GI-NEXT: csel x13, x13, x14, lo
+; CHECK-GI-NEXT: tst x16, #0x7f
+; CHECK-GI-NEXT: lsr x16, x3, x12
+; CHECK-GI-NEXT: lsl x11, x2, x11
+; CHECK-GI-NEXT: csel x13, x15, x13, eq
+; CHECK-GI-NEXT: cmp x17, #64
+; CHECK-GI-NEXT: lsr x15, x2, x12
+; CHECK-GI-NEXT: mvn x9, x9
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: orr x11, x16, x11
; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: orr x9, x1, x9
-; CHECK-GI-NEXT: lsr x18, x3, x12
-; CHECK-GI-NEXT: orr x0, x8, x16
-; CHECK-GI-NEXT: csel x9, x9, x11, lo
-; CHECK-GI-NEXT: tst x10, #0x7f
-; CHECK-GI-NEXT: orr x1, x15, x17
-; CHECK-GI-NEXT: csel x9, x4, x9, eq
+; CHECK-GI-NEXT: csel x11, x11, x15, lo
+; CHECK-GI-NEXT: tst x9, #0x7f
+; CHECK-GI-NEXT: orr x0, x0, x13
+; CHECK-GI-NEXT: csel x9, x3, x11, eq
; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: csel x10, x18, xzr, lo
-; CHECK-GI-NEXT: orr x2, x14, x9
-; CHECK-GI-NEXT: orr x3, x13, x10
-; CHECK-GI-NEXT: ldr x19, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT: orr x1, x8, x14
+; CHECK-GI-NEXT: csel x11, x15, xzr, lo
+; CHECK-GI-NEXT: orr x2, x18, x9
+; CHECK-GI-NEXT: orr x3, x10, x11
; CHECK-GI-NEXT: ret
entry:
%d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
@@ -2982,67 +2938,59 @@ define <2 x i128> @fshr_v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
; CHECK-GI-NEXT: extr x14, x1, x0, #63
; CHECK-GI-NEXT: ldr x8, [sp, #16]
; CHECK-GI-NEXT: bic x11, x10, x9
-; CHECK-GI-NEXT: mvn x16, x9
-; CHECK-GI-NEXT: and x15, x9, #0x7f
+; CHECK-GI-NEXT: lsl x1, x2, #1
+; CHECK-GI-NEXT: extr x2, x3, x2, #63
; CHECK-GI-NEXT: sub x17, x12, x11
-; CHECK-GI-NEXT: sub x18, x11, #64
; CHECK-GI-NEXT: lsl x0, x14, x11
-; CHECK-GI-NEXT: lsr x17, x13, x17
-; CHECK-GI-NEXT: lsl x1, x13, x11
-; CHECK-GI-NEXT: lsl x13, x13, x18
; CHECK-GI-NEXT: bic x10, x10, x8
-; CHECK-GI-NEXT: lsl x18, x2, #1
+; CHECK-GI-NEXT: lsr x17, x13, x17
+; CHECK-GI-NEXT: lsl x13, x13, x11
; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: mvn x18, x9
+; CHECK-GI-NEXT: and x15, x9, #0x7f
+; CHECK-GI-NEXT: and x16, x8, #0x7f
; CHECK-GI-NEXT: orr x17, x17, x0
-; CHECK-GI-NEXT: extr x11, x3, x2, #63
-; CHECK-GI-NEXT: csel x0, x1, xzr, lo
+; CHECK-GI-NEXT: csel x11, x13, xzr, lo
+; CHECK-GI-NEXT: mvn x0, x8
; CHECK-GI-NEXT: csel x13, x17, x13, lo
; CHECK-GI-NEXT: sub x17, x12, x10
-; CHECK-GI-NEXT: tst x16, #0x7f
-; CHECK-GI-NEXT: sub x16, x10, #64
-; CHECK-GI-NEXT: lsr x17, x18, x17
-; CHECK-GI-NEXT: lsl x2, x11, x10
-; CHECK-GI-NEXT: lsl x1, x18, x10
+; CHECK-GI-NEXT: tst x18, #0x7f
+; CHECK-GI-NEXT: lsr x17, x1, x17
+; CHECK-GI-NEXT: lsl x18, x2, x10
; CHECK-GI-NEXT: csel x13, x14, x13, eq
-; CHECK-GI-NEXT: lsl x14, x18, x16
+; CHECK-GI-NEXT: lsl x14, x1, x10
; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: mvn x10, x8
-; CHECK-GI-NEXT: orr x16, x17, x2
-; CHECK-GI-NEXT: csel x17, x1, xzr, lo
-; CHECK-GI-NEXT: csel x14, x16, x14, lo
-; CHECK-GI-NEXT: tst x10, #0x7f
; CHECK-GI-NEXT: sub x10, x12, x15
-; CHECK-GI-NEXT: sub x16, x15, #64
-; CHECK-GI-NEXT: lsr x18, x4, x15
+; CHECK-GI-NEXT: orr x17, x17, x18
+; CHECK-GI-NEXT: lsr x1, x4, x15
; CHECK-GI-NEXT: lsl x10, x5, x10
-; CHECK-GI-NEXT: csel x11, x11, x14, eq
-; CHECK-GI-NEXT: lsr x14, x5, x16
-; CHECK-GI-NEXT: and x1, x8, #0x7f
+; CHECK-GI-NEXT: csel x18, x14, xzr, lo
+; CHECK-GI-NEXT: csel x14, x17, x14, lo
+; CHECK-GI-NEXT: tst x0, #0x7f
+; CHECK-GI-NEXT: lsr x17, x5, x15
+; CHECK-GI-NEXT: csel x14, x2, x14, eq
+; CHECK-GI-NEXT: orr x10, x1, x10
; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: lsr x16, x5, x15
-; CHECK-GI-NEXT: orr x10, x18, x10
-; CHECK-GI-NEXT: csel x10, x10, x14, lo
+; CHECK-GI-NEXT: sub x12, x12, x16
+; CHECK-GI-NEXT: csel x10, x10, x17, lo
; CHECK-GI-NEXT: tst x9, #0x7f
-; CHECK-GI-NEXT: sub x9, x12, x1
-; CHECK-GI-NEXT: sub x12, x1, #64
-; CHECK-GI-NEXT: lsr x14, x6, x1
-; CHECK-GI-NEXT: lsl x9, x7, x9
+; CHECK-GI-NEXT: lsr x9, x6, x16
+; CHECK-GI-NEXT: lsl x12, x7, x12
; CHECK-GI-NEXT: csel x10, x4, x10, eq
; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: lsr x12, x7, x12
-; CHECK-GI-NEXT: csel x15, x16, xzr, lo
-; CHECK-GI-NEXT: orr x9, x14, x9
-; CHECK-GI-NEXT: cmp x1, #64
-; CHECK-GI-NEXT: lsr x14, x7, x1
-; CHECK-GI-NEXT: csel x9, x9, x12, lo
+; CHECK-GI-NEXT: lsr x15, x7, x16
+; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: cmp x16, #64
+; CHECK-GI-NEXT: orr x9, x9, x12
+; CHECK-GI-NEXT: orr x0, x11, x10
+; CHECK-GI-NEXT: orr x1, x13, x17
+; CHECK-GI-NEXT: csel x9, x9, x15, lo
; CHECK-GI-NEXT: tst x8, #0x7f
; CHECK-GI-NEXT: csel x8, x6, x9, eq
-; CHECK-GI-NEXT: cmp x1, #64
-; CHECK-GI-NEXT: orr x0, x0, x10
-; CHECK-GI-NEXT: csel x9, x14, xzr, lo
-; CHECK-GI-NEXT: orr x1, x13, x15
-; CHECK-GI-NEXT: orr x2, x17, x8
-; CHECK-GI-NEXT: orr x3, x11, x9
+; CHECK-GI-NEXT: cmp x16, #64
+; CHECK-GI-NEXT: csel x9, x15, xzr, lo
+; CHECK-GI-NEXT: orr x2, x18, x8
+; CHECK-GI-NEXT: orr x3, x14, x9
; CHECK-GI-NEXT: ret
entry:
%d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
diff --git a/llvm/test/CodeGen/AArch64/funnel-shift.ll b/llvm/test/CodeGen/AArch64/funnel-shift.ll
index c4e4d3c7dcd2b9..d5233d7e7699c4 100644
--- a/llvm/test/CodeGen/AArch64/funnel-shift.ll
+++ b/llvm/test/CodeGen/AArch64/funnel-shift.ll
@@ -89,35 +89,31 @@ define i128 @fshl_i128(i128 %x, i128 %y, i128 %z) nounwind {
; CHECK-GI-NEXT: and x9, x4, #0x7f
; CHECK-GI-NEXT: mov w10, #127 // =0x7f
; CHECK-GI-NEXT: sub x12, x8, x9
-; CHECK-GI-NEXT: lsl x13, x1, x9
; CHECK-GI-NEXT: bic x10, x10, x4
+; CHECK-GI-NEXT: lsl x13, x1, x9
; CHECK-GI-NEXT: lsr x12, x0, x12
-; CHECK-GI-NEXT: sub x14, x9, #64
+; CHECK-GI-NEXT: extr x14, x3, x2, #1
+; CHECK-GI-NEXT: lsr x16, x3, #1
; CHECK-GI-NEXT: lsl x15, x0, x9
-; CHECK-GI-NEXT: extr x16, x3, x2, #1
-; CHECK-GI-NEXT: cmp x9, #64
; CHECK-GI-NEXT: sub x8, x8, x10
+; CHECK-GI-NEXT: cmp x9, #64
; CHECK-GI-NEXT: orr x9, x12, x13
-; CHECK-GI-NEXT: lsr x12, x3, #1
-; CHECK-GI-NEXT: lsl x13, x0, x14
-; CHECK-GI-NEXT: csel x14, x15, xzr, lo
-; CHECK-GI-NEXT: sub x15, x10, #64
-; CHECK-GI-NEXT: lsr x17, x16, x10
-; CHECK-GI-NEXT: lsl x8, x12, x8
-; CHECK-GI-NEXT: csel x9, x9, x13, lo
+; CHECK-GI-NEXT: lsr x13, x14, x10
+; CHECK-GI-NEXT: lsl x8, x16, x8
+; CHECK-GI-NEXT: csel x12, x15, xzr, lo
+; CHECK-GI-NEXT: csel x9, x9, x15, lo
; CHECK-GI-NEXT: tst x4, #0x7f
-; CHECK-GI-NEXT: lsr x13, x12, x15
+; CHECK-GI-NEXT: lsr x15, x16, x10
; CHECK-GI-NEXT: mvn x11, x4
; CHECK-GI-NEXT: csel x9, x1, x9, eq
-; CHECK-GI-NEXT: orr x8, x17, x8
+; CHECK-GI-NEXT: orr x8, x13, x8
; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: lsr x12, x12, x10
-; CHECK-GI-NEXT: csel x8, x8, x13, lo
+; CHECK-GI-NEXT: csel x8, x8, x15, lo
; CHECK-GI-NEXT: tst x11, #0x7f
-; CHECK-GI-NEXT: csel x8, x16, x8, eq
+; CHECK-GI-NEXT: csel x8, x14, x8, eq
; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: csel x10, x12, xzr, lo
-; CHECK-GI-NEXT: orr x0, x14, x8
+; CHECK-GI-NEXT: csel x10, x15, xzr, lo
+; CHECK-GI-NEXT: orr x0, x12, x8
; CHECK-GI-NEXT: orr x1, x9, x10
; CHECK-GI-NEXT: ret
%f = call i128 @llvm.fshl.i128(i128 %x, i128 %y, i128 %z)
diff --git a/llvm/test/CodeGen/AArch64/shift-mod.ll b/llvm/test/CodeGen/AArch64/shift-mod.ll
index c9333fb757a79a..72c68aee711b37 100644
--- a/llvm/test/CodeGen/AArch64/shift-mod.ll
+++ b/llvm/test/CodeGen/AArch64/shift-mod.ll
@@ -36,16 +36,10 @@ define i64 @test2(i32 %x, i64 %y) {
}
define i64 @test3(i64 %x, i64 %y) {
-; CHECK-SD-LABEL: test3:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: lsl x0, x1, x0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test3:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: add x8, x0, #64
-; CHECK-GI-NEXT: lsl x0, x1, x8
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: test3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl x0, x1, x0
+; CHECK-NEXT: ret
%add = add nsw i64 64, %x
%shl = shl i64 %y, %add
ret i64 %shl
@@ -285,3 +279,34 @@ define i64 @ashr_i64_i16(i64 %x, i16 %amt) {
%r = ashr i64 %x, %ext
ret i64 %r
}
+
+; Test ADD/SUB by multiple of shift size is optimized away.
+define i32 @shl_i32_add64(i32 %x, i32 %amt) {
+; CHECK-LABEL: shl_i32_add64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl w0, w0, w1
+; CHECK-NEXT: ret
+ %add = add i32 %amt, 64
+ %r = shl i32 %x, %add
+ ret i32 %r
+}
+
+define i32 @lshr_i32_add32(i32 %x, i32 %amt) {
+; CHECK-LABEL: lshr_i32_add32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsr w0, w0, w1
+; CHECK-NEXT: ret
+ %add = add i32 %amt, 32
+ %r = lshr i32 %x, %add
+ ret i32 %r
+}
+
+define i64 @shl_i64_add64(i64 %x, i64 %amt) {
+; CHECK-LABEL: shl_i64_add64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl x0, x0, x1
+; CHECK-NEXT: ret
+ %add = add i64 %amt, 64
+ %r = shl i64 %x, %add
+ ret i64 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/shift.ll b/llvm/test/CodeGen/AArch64/shift.ll
index eb5f56f771a0f2..2fcdf4df419310 100644
--- a/llvm/test/CodeGen/AArch64/shift.ll
+++ b/llvm/test/CodeGen/AArch64/shift.ll
@@ -71,16 +71,14 @@ define i128 @shl_i128(i128 %0, i128 %1){
; CHECK-GI-LABEL: shl_i128:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: sub x9, x2, #64
-; CHECK-GI-NEXT: lsl x10, x1, x2
+; CHECK-GI-NEXT: lsl x9, x1, x2
+; CHECK-GI-NEXT: lsl x10, x0, x2
; CHECK-GI-NEXT: sub x8, x8, x2
-; CHECK-GI-NEXT: lsl x11, x0, x2
-; CHECK-GI-NEXT: lsl x9, x0, x9
-; CHECK-GI-NEXT: lsr x8, x0, x8
; CHECK-GI-NEXT: cmp x2, #64
-; CHECK-GI-NEXT: csel x0, x11, xzr, lo
-; CHECK-GI-NEXT: orr x8, x8, x10
-; CHECK-GI-NEXT: csel x8, x8, x9, lo
+; CHECK-GI-NEXT: lsr x8, x0, x8
+; CHECK-GI-NEXT: csel x0, x10, xzr, lo
+; CHECK-GI-NEXT: orr x8, x8, x9
+; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x2, #0
; CHECK-GI-NEXT: csel x1, x1, x8, eq
; CHECK-GI-NEXT: ret
@@ -161,20 +159,18 @@ define i128 @ashr_i128(i128 %0, i128 %1){
; CHECK-GI-LABEL: ashr_i128:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: sub x9, x2, #64
-; CHECK-GI-NEXT: lsr x10, x0, x2
+; CHECK-GI-NEXT: lsr x9, x0, x2
+; CHECK-GI-NEXT: asr x10, x1, x2
; CHECK-GI-NEXT: sub x8, x8, x2
-; CHECK-GI-NEXT: asr x9, x1, x9
; CHECK-GI-NEXT: cmp x2, #64
; CHECK-GI-NEXT: lsl x8, x1, x8
-; CHECK-GI-NEXT: asr x11, x1, x2
-; CHECK-GI-NEXT: orr x8, x10, x8
-; CHECK-GI-NEXT: asr x10, x1, #63
-; CHECK-GI-NEXT: csel x8, x8, x9, lo
+; CHECK-GI-NEXT: orr x8, x9, x8
+; CHECK-GI-NEXT: asr x9, x1, #63
+; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x2, #0
; CHECK-GI-NEXT: csel x0, x0, x8, eq
; CHECK-GI-NEXT: cmp x2, #64
-; CHECK-GI-NEXT: csel x1, x11, x10, lo
+; CHECK-GI-NEXT: csel x1, x10, x9, lo
; CHECK-GI-NEXT: ret
%3 = ashr i128 %0, %1
ret i128 %3
@@ -251,15 +247,13 @@ define i128 @lshr_i128(i128 %0, i128 %1){
; CHECK-GI-LABEL: lshr_i128:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: sub x9, x2, #64
-; CHECK-GI-NEXT: lsr x10, x0, x2
+; CHECK-GI-NEXT: lsr x9, x0, x2
+; CHECK-GI-NEXT: lsr x10, x1, x2
; CHECK-GI-NEXT: sub x8, x8, x2
-; CHECK-GI-NEXT: lsr x9, x1, x9
; CHECK-GI-NEXT: cmp x2, #64
; CHECK-GI-NEXT: lsl x8, x1, x8
-; CHECK-GI-NEXT: orr x8, x10, x8
-; CHECK-GI-NEXT: lsr x10, x1, x2
-; CHECK-GI-NEXT: csel x8, x8, x9, lo
+; CHECK-GI-NEXT: orr x8, x9, x8
+; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x2, #0
; CHECK-GI-NEXT: csel x0, x0, x8, eq
; CHECK-GI-NEXT: cmp x2, #64
@@ -620,28 +614,24 @@ define <2 x i128> @shl_v2i128(<2 x i128> %0, <2 x i128> %1){
; CHECK-GI-LABEL: shl_v2i128:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: sub x10, x4, #64
-; CHECK-GI-NEXT: lsl x11, x1, x4
+; CHECK-GI-NEXT: lsl x10, x1, x4
+; CHECK-GI-NEXT: lsl x11, x0, x4
; CHECK-GI-NEXT: sub x9, x8, x4
-; CHECK-GI-NEXT: lsl x10, x0, x10
-; CHECK-GI-NEXT: lsl x12, x0, x4
-; CHECK-GI-NEXT: lsr x9, x0, x9
-; CHECK-GI-NEXT: cmp x4, #64
; CHECK-GI-NEXT: sub x8, x8, x6
+; CHECK-GI-NEXT: cmp x4, #64
+; CHECK-GI-NEXT: lsr x9, x0, x9
; CHECK-GI-NEXT: lsr x8, x2, x8
-; CHECK-GI-NEXT: csel x0, x12, xzr, lo
-; CHECK-GI-NEXT: lsl x12, x2, x6
-; CHECK-GI-NEXT: orr x9, x9, x11
-; CHECK-GI-NEXT: lsl x11, x3, x6
-; CHECK-GI-NEXT: csel x9, x9, x10, lo
-; CHECK-GI-NEXT: sub x10, x6, #64
+; CHECK-GI-NEXT: csel x0, x11, xzr, lo
+; CHECK-GI-NEXT: orr x9, x9, x10
+; CHECK-GI-NEXT: lsl x10, x3, x6
+; CHECK-GI-NEXT: csel x9, x9, x11, lo
; CHECK-GI-NEXT: cmp x4, #0
-; CHECK-GI-NEXT: lsl x10, x2, x10
+; CHECK-GI-NEXT: lsl x11, x2, x6
; CHECK-GI-NEXT: csel x1, x1, x9, eq
-; CHECK-GI-NEXT: orr x8, x8, x11
+; CHECK-GI-NEXT: orr x8, x8, x10
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: csel x2, x12, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x10, lo
+; CHECK-GI-NEXT: csel x2, x11, xzr, lo
+; CHECK-GI-NEXT: csel x8, x8, x11, lo
; CHECK-GI-NEXT: cmp x6, #0
; CHECK-GI-NEXT: csel x3, x3, x8, eq
; CHECK-GI-NEXT: ret
@@ -788,34 +778,30 @@ define <2 x i128> @ashr_v2i128(<2 x i128> %0, <2 x i128> %1){
; CHECK-GI-LABEL: ashr_v2i128:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: sub x10, x4, #64
-; CHECK-GI-NEXT: lsr x11, x0, x4
+; CHECK-GI-NEXT: lsr x10, x0, x4
+; CHECK-GI-NEXT: asr x11, x1, x4
; CHECK-GI-NEXT: sub x9, x8, x4
-; CHECK-GI-NEXT: asr x10, x1, x10
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: lsl x9, x1, x9
; CHECK-GI-NEXT: sub x8, x8, x6
-; CHECK-GI-NEXT: asr x12, x1, x4
+; CHECK-GI-NEXT: lsl x9, x1, x9
+; CHECK-GI-NEXT: lsr x12, x2, x6
; CHECK-GI-NEXT: lsl x8, x3, x8
-; CHECK-GI-NEXT: orr x9, x11, x9
-; CHECK-GI-NEXT: asr x11, x1, #63
-; CHECK-GI-NEXT: csel x9, x9, x10, lo
+; CHECK-GI-NEXT: orr x9, x10, x9
+; CHECK-GI-NEXT: asr x10, x1, #63
+; CHECK-GI-NEXT: orr x8, x12, x8
+; CHECK-GI-NEXT: csel x9, x9, x11, lo
; CHECK-GI-NEXT: cmp x4, #0
-; CHECK-GI-NEXT: lsr x10, x2, x6
; CHECK-GI-NEXT: csel x0, x0, x9, eq
-; CHECK-GI-NEXT: sub x9, x6, #64
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: asr x9, x3, x9
-; CHECK-GI-NEXT: csel x1, x12, x11, lo
-; CHECK-GI-NEXT: orr x8, x10, x8
+; CHECK-GI-NEXT: asr x9, x3, x6
+; CHECK-GI-NEXT: csel x1, x11, x10, lo
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: asr x11, x3, x6
; CHECK-GI-NEXT: asr x10, x3, #63
; CHECK-GI-NEXT: csel x8, x8, x9, lo
; CHECK-GI-NEXT: cmp x6, #0
; CHECK-GI-NEXT: csel x2, x2, x8, eq
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: csel x3, x11, x10, lo
+; CHECK-GI-NEXT: csel x3, x9, x10, lo
; CHECK-GI-NEXT: ret
%3 = ashr <2 x i128> %0, %1
ret <2 x i128> %3
@@ -956,32 +942,28 @@ define <2 x i128> @lshr_v2i128(<2 x i128> %0, <2 x i128> %1){
; CHECK-GI-LABEL: lshr_v2i128:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: sub x10, x4, #64
-; CHECK-GI-NEXT: lsr x11, x0, x4
+; CHECK-GI-NEXT: lsr x10, x0, x4
+; CHECK-GI-NEXT: lsr x11, x1, x4
; CHECK-GI-NEXT: sub x9, x8, x4
-; CHECK-GI-NEXT: lsr x10, x1, x10
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: lsl x9, x1, x9
; CHECK-GI-NEXT: sub x8, x8, x6
-; CHECK-GI-NEXT: lsr x12, x1, x4
+; CHECK-GI-NEXT: lsl x9, x1, x9
; CHECK-GI-NEXT: lsl x8, x3, x8
-; CHECK-GI-NEXT: orr x9, x11, x9
-; CHECK-GI-NEXT: lsr x11, x2, x6
-; CHECK-GI-NEXT: csel x9, x9, x10, lo
+; CHECK-GI-NEXT: orr x9, x10, x9
+; CHECK-GI-NEXT: lsr x10, x2, x6
+; CHECK-GI-NEXT: csel x9, x9, x11, lo
; CHECK-GI-NEXT: cmp x4, #0
-; CHECK-GI-NEXT: sub x10, x6, #64
; CHECK-GI-NEXT: csel x0, x0, x9, eq
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: lsr x9, x3, x10
-; CHECK-GI-NEXT: csel x1, x12, xzr, lo
-; CHECK-GI-NEXT: orr x8, x11, x8
+; CHECK-GI-NEXT: lsr x9, x3, x6
+; CHECK-GI-NEXT: csel x1, x11, xzr, lo
+; CHECK-GI-NEXT: orr x8, x10, x8
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: lsr x10, x3, x6
; CHECK-GI-NEXT: csel x8, x8, x9, lo
; CHECK-GI-NEXT: cmp x6, #0
; CHECK-GI-NEXT: csel x2, x2, x8, eq
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: csel x3, x10, xzr, lo
+; CHECK-GI-NEXT: csel x3, x9, xzr, lo
; CHECK-GI-NEXT: ret
%3 = lshr <2 x i128> %0, %1
ret <2 x i128> %3
>From acf68b4e5cba58aa8eb2214b9b21634c6bfbd249 Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Thu, 24 Sep 2026 16:11:12 +0530
Subject: [PATCH 3/3] AArch64: Generate NEG for shift by (N - X) in
SelectShiftMask
Extend SelectShiftMask to handle SUB where the constant operand is a
multiple of the shift width: shl x, (sub N, y) where N % ShiftWidth == 0
becomes shl x, neg(y), generating a NEG instead of a SUB from a constant.
This implements both SelectionDAG (SelectShiftMask) and GlobalISel
(selectShiftMask). The GlobalISel side builds the NEG (SUBWrr/SUBXrr with
WZR/XZR) in the ComplexPattern renderer and constrains its operands.
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 20 ++++++++++
.../GISel/AArch64InstructionSelector.cpp | 24 ++++++++++++
llvm/test/CodeGen/AArch64/shift-mod.ll | 37 +++++++++++++++++++
3 files changed, 81 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 1a991ffcbdeebd..4c5925e1d99fbd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -798,6 +798,26 @@ bool AArch64DAGToDAGISel::SelectShiftMask(SDValue N, SDValue &ShAmt) {
}
}
+ // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
+ // to generate a NEG instead of a SUB from a constant.
+ if (N.getOpcode() == ISD::SUB && N.hasOneUse() &&
+ N.getValueType() == (ShiftWidth == 32 ? MVT::i32 : MVT::i64)) {
+ uint64_t Imm;
+ if (isIntImmediate(N.getOperand(0).getNode(), Imm) && Imm != 0 &&
+ (Imm % ShiftWidth == 0)) {
+ SDLoc DL(N);
+ EVT VT = N.getValueType();
+ unsigned NegOpc = (ShiftWidth == 32) ? AArch64::SUBWrr : AArch64::SUBXrr;
+ unsigned ZeroReg = (ShiftWidth == 32) ? AArch64::WZR : AArch64::XZR;
+ SDValue Zero =
+ CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, VT);
+ MachineSDNode *Neg =
+ CurDAG->getMachineNode(NegOpc, DL, VT, Zero, N.getOperand(1));
+ ShAmt = SDValue(Neg, 0);
+ return true;
+ }
+ }
+
return false;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index de66ab60a9abd7..26bf3d4ecb533c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -7352,6 +7352,30 @@ AArch64InstructionSelector::selectShiftMask(MachineOperand &Root) const {
return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
}
+ // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
+ // to generate a NEG instead of a SUB from a constant.
+ Register SubSrcReg;
+ int64_t SubImm;
+ if (MRI.hasOneUse(ShAmtReg) &&
+ mi_match(ShAmtReg, MRI, m_GSub(m_ICst(SubImm), m_Reg(SubSrcReg))) &&
+ SubImm != 0 && (SubImm % ShiftWidth == 0)) {
+ return {{[=](MachineInstrBuilder &MIB) {
+ MachineInstr *I = MIB.getInstr();
+ MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
+ const TargetRegisterClass &RC =
+ ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
+ unsigned SubOpc = ShiftWidth == 32 ? AArch64::SUBWrr : AArch64::SUBXrr;
+ Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
+ Register NegReg = MRI2.createVirtualRegister(&RC);
+ auto NegMI = BuildMI(*I->getParent(), *I, I->getDebugLoc(),
+ TII.get(SubOpc), NegReg)
+ .addReg(ZeroReg)
+ .addReg(SubSrcReg);
+ constrainSelectedInstRegOperands(*NegMI, TII, TRI, RBI);
+ MIB.addReg(NegReg);
+ }}};
+ }
+
return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
}
diff --git a/llvm/test/CodeGen/AArch64/shift-mod.ll b/llvm/test/CodeGen/AArch64/shift-mod.ll
index 72c68aee711b37..303a6d5b1f9599 100644
--- a/llvm/test/CodeGen/AArch64/shift-mod.ll
+++ b/llvm/test/CodeGen/AArch64/shift-mod.ll
@@ -310,3 +310,40 @@ define i64 @shl_i64_add64(i64 %x, i64 %amt) {
%r = shl i64 %x, %add
ret i64 %r
}
+
+; Test SUB N-X where N == 0 mod size generates NEG.
+define i32 @shl_i32_sub64(i32 %x, i32 %amt) {
+; CHECK-SD-LABEL: shl_i32_sub64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: neg w8, w1
+; CHECK-SD-NEXT: lsl w0, w0, w8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: shl_i32_sub64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w8, #64 // =0x40
+; CHECK-GI-NEXT: sub w8, w8, w1
+; CHECK-GI-NEXT: lsl w0, w0, w8
+; CHECK-GI-NEXT: ret
+ %sub = sub i32 64, %amt
+ %r = shl i32 %x, %sub
+ ret i32 %r
+}
+
+define i64 @lshr_i64_sub64(i64 %x, i64 %amt) {
+; CHECK-SD-LABEL: lshr_i64_sub64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: neg x8, x1
+; CHECK-SD-NEXT: lsr x0, x0, x8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: lshr_i64_sub64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w8, #64 // =0x40
+; CHECK-GI-NEXT: sub x8, x8, x1
+; CHECK-GI-NEXT: lsr x0, x0, x8
+; CHECK-GI-NEXT: ret
+ %sub = sub i64 64, %amt
+ %r = lshr i64 %x, %sub
+ ret i64 %r
+}
More information about the llvm-commits
mailing list