[llvm] [AArch64] Remove tryShiftAmountMod - gap finding (PR #228753)
Deepak Shirke via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 12:04:56 PDT 2026
https://github.com/deepakshirkem updated https://github.com/llvm/llvm-project/pull/228753
>From 599d7b1fb4356aee24a3f9ebd2953b156abf5ceb Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Thu, 24 Sep 2026 16:11:12 +0530
Subject: [PATCH 1/4] AArch64: Generate NEG for shift by (N - X) in
SelectShiftMask
Extend SelectShiftMask to handle SUB where the constant operand is a
multiple of the shift width: shl x, (sub N, y) where N % ShiftWidth == 0
becomes shl x, neg(y), generating a NEG instead of a SUB from a constant.
This implements both SelectionDAG (SelectShiftMask) and GlobalISel
(selectShiftMask). The GlobalISel side builds the NEG (SUBWrr/SUBXrr with
WZR/XZR) in the ComplexPattern renderer and constrains its operands.
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 20 +
.../GISel/AArch64InstructionSelector.cpp | 24 +
.../test/CodeGen/AArch64/and-mask-variable.ll | 13 +-
llvm/test/CodeGen/AArch64/fcvt-i256.ll | 488 +++++++------
llvm/test/CodeGen/AArch64/fsh.ll | 655 +++++++++---------
llvm/test/CodeGen/AArch64/funnel-shift.ll | 197 +++---
llvm/test/CodeGen/AArch64/shift-mod.ll | 73 +-
llvm/test/CodeGen/AArch64/shift.ll | 112 ++-
8 files changed, 835 insertions(+), 747 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 962340bd9a4ce..31bedd86859a9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -802,6 +802,26 @@ bool AArch64DAGToDAGISel::SelectShiftMask(SDValue N, SDValue &ShAmt) {
}
}
+ // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
+ // to generate a NEG instead of a SUB from a constant.
+ if (N.getOpcode() == ISD::SUB && N.hasOneUse() &&
+ N.getValueType() == (ShiftWidth == 32 ? MVT::i32 : MVT::i64)) {
+ uint64_t Imm;
+ if (isIntImmediate(N.getOperand(0).getNode(), Imm) && Imm != 0 &&
+ (Imm % ShiftWidth == 0)) {
+ SDLoc DL(N);
+ EVT VT = N.getValueType();
+ unsigned NegOpc = (ShiftWidth == 32) ? AArch64::SUBWrr : AArch64::SUBXrr;
+ unsigned ZeroReg = (ShiftWidth == 32) ? AArch64::WZR : AArch64::XZR;
+ SDValue Zero =
+ CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, VT);
+ MachineSDNode *Neg =
+ CurDAG->getMachineNode(NegOpc, DL, VT, Zero, N.getOperand(1));
+ ShAmt = SDValue(Neg, 0);
+ return true;
+ }
+ }
+
return false;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index e12acd42bfdfd..29c1f32911d3c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -7353,6 +7353,30 @@ AArch64InstructionSelector::selectShiftMask(MachineOperand &Root) const {
return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
}
+ // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
+ // to generate a NEG instead of a SUB from a constant.
+ Register SubSrcReg;
+ int64_t SubImm;
+ if (MRI.hasOneUse(ShAmtReg) &&
+ mi_match(ShAmtReg, MRI, m_GSub(m_ICst(SubImm), m_Reg(SubSrcReg))) &&
+ SubImm != 0 && (SubImm % ShiftWidth == 0)) {
+ return {{[=](MachineInstrBuilder &MIB) {
+ MachineInstr *I = MIB.getInstr();
+ MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
+ const TargetRegisterClass &RC =
+ ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
+ unsigned SubOpc = ShiftWidth == 32 ? AArch64::SUBWrr : AArch64::SUBXrr;
+ Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
+ Register NegReg = MRI2.createVirtualRegister(&RC);
+ auto NegMI = BuildMI(*I->getParent(), *I, I->getDebugLoc(),
+ TII.get(SubOpc), NegReg)
+ .addReg(ZeroReg)
+ .addReg(SubSrcReg);
+ constrainSelectedInstRegOperands(*NegMI, TII, TRI, RBI);
+ MIB.addReg(NegReg);
+ }}};
+ }
+
// Only succeed if we changed the shift amount; otherwise let other
// patterns (e.g. zext GPR32 -> SUBREG_TO_REG) match instead.
if (ShAmtReg == Root.getReg())
diff --git a/llvm/test/CodeGen/AArch64/and-mask-variable.ll b/llvm/test/CodeGen/AArch64/and-mask-variable.ll
index ea5079770b243..bcdff64e592a3 100644
--- a/llvm/test/CodeGen/AArch64/and-mask-variable.ll
+++ b/llvm/test/CodeGen/AArch64/and-mask-variable.ll
@@ -56,17 +56,16 @@ define i128 @mask_pair_128(i128 %x, i128 %y) {
;
; CHECK-GI-LABEL: mask_pair_128:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: mov x9, #-1 // =0xffffffffffffffff
+; CHECK-GI-NEXT: mov x8, #-1 // =0xffffffffffffffff
+; CHECK-GI-NEXT: neg x9, x2
; CHECK-GI-NEXT: cmp x2, #64
-; CHECK-GI-NEXT: sub x8, x8, x2
-; CHECK-GI-NEXT: lsl x10, x9, x2
-; CHECK-GI-NEXT: lsr x8, x9, x8
-; CHECK-GI-NEXT: csel x9, x10, xzr, lo
+; CHECK-GI-NEXT: lsl x10, x8, x2
+; CHECK-GI-NEXT: lsr x8, x8, x9
; CHECK-GI-NEXT: orr x8, x8, x10
-; CHECK-GI-NEXT: and x0, x9, x0
+; CHECK-GI-NEXT: csel x9, x10, xzr, lo
; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x2, #0
+; CHECK-GI-NEXT: and x0, x9, x0
; CHECK-GI-NEXT: csinv x8, x8, xzr, ne
; CHECK-GI-NEXT: and x1, x8, x1
; CHECK-GI-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/fcvt-i256.ll b/llvm/test/CodeGen/AArch64/fcvt-i256.ll
index 10b16c0b205d5..67c75ffd70f80 100644
--- a/llvm/test/CodeGen/AArch64/fcvt-i256.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt-i256.ll
@@ -698,69 +698,67 @@ define i256 @f32_to_s256(float %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB4_3: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub w11, w11, #150
-; CHECK-GI-NEXT: mov w13, #64 // =0x40
; CHECK-GI-NEXT: mov w12, #128 // =0x80
-; CHECK-GI-NEXT: sub x15, x13, x11
-; CHECK-GI-NEXT: lsl x16, x10, x11
-; CHECK-GI-NEXT: lsl x17, x10, x11
-; CHECK-GI-NEXT: lsr x15, x10, x15
+; CHECK-GI-NEXT: neg x14, x11
+; CHECK-GI-NEXT: lsl x15, x10, x11
; CHECK-GI-NEXT: sub x12, x12, x11
+; CHECK-GI-NEXT: lsr x14, x10, x14
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x14, x11, #128
-; CHECK-GI-NEXT: lsr x18, x10, x12
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x15, x15, x17, lo
+; CHECK-GI-NEXT: sub x13, x11, #128
+; CHECK-GI-NEXT: lsr x16, x10, x12
+; CHECK-GI-NEXT: csel x18, x15, xzr, lo
+; CHECK-GI-NEXT: lsl x17, x10, x13
+; CHECK-GI-NEXT: csel x14, x14, x15, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: sub x13, x13, x14
-; CHECK-GI-NEXT: csel x15, xzr, x15, eq
+; CHECK-GI-NEXT: neg x15, x13
+; CHECK-GI-NEXT: csel x14, xzr, x14, eq
; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: lsl x0, x10, x14
-; CHECK-GI-NEXT: lsr x13, x10, x13
-; CHECK-GI-NEXT: csel x18, x18, xzr, lo
+; CHECK-GI-NEXT: lsr x15, x10, x15
+; CHECK-GI-NEXT: csel x16, x16, xzr, lo
; CHECK-GI-NEXT: cmp x12, #0
-; CHECK-GI-NEXT: csel x12, x10, x18, eq
-; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: csel x17, x17, xzr, lo
-; CHECK-GI-NEXT: csel x10, x13, x0, lo
-; CHECK-GI-NEXT: cmp x14, #0
+; CHECK-GI-NEXT: csel x12, x10, x16, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x16, x17, xzr, lo
+; CHECK-GI-NEXT: csel x10, x15, x17, lo
+; CHECK-GI-NEXT: cmp x13, #0
; CHECK-GI-NEXT: csel x13, xzr, x10, eq
; CHECK-GI-NEXT: cmp x11, #128
-; CHECK-GI-NEXT: csel x10, x16, xzr, lo
-; CHECK-GI-NEXT: csel x15, x15, xzr, lo
-; CHECK-GI-NEXT: csel x12, x12, x17, lo
+; CHECK-GI-NEXT: csel x10, x18, xzr, lo
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: csel x12, x12, x16, lo
; CHECK-GI-NEXT: csel x13, xzr, x13, lo
; CHECK-GI-NEXT: cmp x11, #0
; CHECK-GI-NEXT: umulh x18, x10, x9
; CHECK-GI-NEXT: csel x12, xzr, x12, eq
; CHECK-GI-NEXT: csel x13, xzr, x13, eq
-; CHECK-GI-NEXT: umulh x11, x15, x9
-; CHECK-GI-NEXT: umulh x17, x12, x8
-; CHECK-GI-NEXT: umulh x14, x10, x8
+; CHECK-GI-NEXT: umulh x11, x14, x9
+; CHECK-GI-NEXT: umulh x16, x12, x8
+; CHECK-GI-NEXT: umulh x15, x10, x8
; CHECK-GI-NEXT: add x11, x11, x18
-; CHECK-GI-NEXT: mul x16, x10, x9
-; CHECK-GI-NEXT: add x11, x17, x11
-; CHECK-GI-NEXT: mul x0, x15, x8
-; CHECK-GI-NEXT: umulh x2, x15, x8
-; CHECK-GI-NEXT: adds x14, x14, x16
+; CHECK-GI-NEXT: mul x17, x10, x9
+; CHECK-GI-NEXT: add x11, x16, x11
+; CHECK-GI-NEXT: mul x0, x14, x8
+; CHECK-GI-NEXT: umulh x2, x14, x8
; CHECK-GI-NEXT: madd x11, x10, x9, x11
+; CHECK-GI-NEXT: mul x16, x14, x9
+; CHECK-GI-NEXT: madd x11, x14, x9, x11
+; CHECK-GI-NEXT: adds x14, x15, x17
; CHECK-GI-NEXT: cset w3, hs
; CHECK-GI-NEXT: adds x1, x14, x0
-; CHECK-GI-NEXT: mul x17, x15, x9
+; CHECK-GI-NEXT: mul x15, x12, x8
; CHECK-GI-NEXT: cset w14, hs
; CHECK-GI-NEXT: adds x18, x2, x18
-; CHECK-GI-NEXT: madd x11, x15, x9, x11
; CHECK-GI-NEXT: cset w0, hs
-; CHECK-GI-NEXT: adds x16, x18, x16
-; CHECK-GI-NEXT: and x18, x0, #0x1
-; CHECK-GI-NEXT: mul x15, x12, x8
+; CHECK-GI-NEXT: adds x17, x18, x17
; CHECK-GI-NEXT: madd x9, x12, x9, x11
; CHECK-GI-NEXT: and x11, x3, #0x1
; CHECK-GI-NEXT: and x12, x14, #0x1
; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: and x18, x0, #0x1
; CHECK-GI-NEXT: and x14, x14, #0x1
; CHECK-GI-NEXT: mul x0, x10, x8
; CHECK-GI-NEXT: add x10, x11, x12
-; CHECK-GI-NEXT: adds x12, x16, x17
+; CHECK-GI-NEXT: adds x12, x17, x16
; CHECK-GI-NEXT: add x11, x18, x14
; CHECK-GI-NEXT: cset w14, hs
; CHECK-GI-NEXT: adds x12, x12, x15
@@ -868,36 +866,34 @@ define i256 @f32_to_u256(float %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB5_3: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub w9, w9, #150
-; CHECK-GI-NEXT: mov w11, #64 // =0x40
; CHECK-GI-NEXT: mov w10, #128 // =0x80
-; CHECK-GI-NEXT: sub x12, x11, x9
-; CHECK-GI-NEXT: lsl x13, x8, x9
-; CHECK-GI-NEXT: lsl x14, x8, x9
-; CHECK-GI-NEXT: lsr x12, x8, x12
+; CHECK-GI-NEXT: neg x11, x9
+; CHECK-GI-NEXT: lsl x12, x8, x9
; CHECK-GI-NEXT: sub x10, x10, x9
+; CHECK-GI-NEXT: lsr x11, x8, x11
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsr x15, x8, x10
-; CHECK-GI-NEXT: csel x13, x13, xzr, lo
-; CHECK-GI-NEXT: sub x16, x9, #128
-; CHECK-GI-NEXT: csel x12, x12, x14, lo
+; CHECK-GI-NEXT: lsr x13, x8, x10
+; CHECK-GI-NEXT: csel x14, x12, xzr, lo
+; CHECK-GI-NEXT: csel x11, x11, x12, lo
; CHECK-GI-NEXT: cmp x9, #0
-; CHECK-GI-NEXT: sub x11, x11, x16
-; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: sub x12, x9, #128
+; CHECK-GI-NEXT: csel x11, xzr, x11, eq
; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: lsr x11, x8, x11
-; CHECK-GI-NEXT: csel x15, x15, xzr, lo
+; CHECK-GI-NEXT: neg x15, x12
+; CHECK-GI-NEXT: csel x13, x13, xzr, lo
; CHECK-GI-NEXT: cmp x10, #0
-; CHECK-GI-NEXT: lsl x10, x8, x16
-; CHECK-GI-NEXT: csel x8, x8, x15, eq
-; CHECK-GI-NEXT: cmp x16, #64
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x10, x11, x10, lo
-; CHECK-GI-NEXT: cmp x16, #0
+; CHECK-GI-NEXT: lsl x10, x8, x12
+; CHECK-GI-NEXT: lsr x15, x8, x15
+; CHECK-GI-NEXT: csel x8, x8, x13, eq
+; CHECK-GI-NEXT: cmp x12, #64
+; CHECK-GI-NEXT: csel x13, x10, xzr, lo
+; CHECK-GI-NEXT: csel x10, x15, x10, lo
+; CHECK-GI-NEXT: cmp x12, #0
; CHECK-GI-NEXT: csel x10, xzr, x10, eq
; CHECK-GI-NEXT: cmp x9, #128
-; CHECK-GI-NEXT: csel x0, x13, xzr, lo
-; CHECK-GI-NEXT: csel x1, x12, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x14, lo
+; CHECK-GI-NEXT: csel x0, x14, xzr, lo
+; CHECK-GI-NEXT: csel x1, x11, xzr, lo
+; CHECK-GI-NEXT: csel x8, x8, x13, lo
; CHECK-GI-NEXT: csel x10, xzr, x10, lo
; CHECK-GI-NEXT: cmp x9, #0
; CHECK-GI-NEXT: csel x2, xzr, x8, eq
@@ -1053,77 +1049,76 @@ define i256 @f64_to_s256(double %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB6_3: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub x11, x11, #1075
-; CHECK-GI-NEXT: mov w12, #64 // =0x40
-; CHECK-GI-NEXT: mov w13, #128 // =0x80
-; CHECK-GI-NEXT: sub x15, x12, x11
-; CHECK-GI-NEXT: lsl x16, x10, x11
-; CHECK-GI-NEXT: sub x13, x13, x11
-; CHECK-GI-NEXT: lsr x15, x10, x15
+; CHECK-GI-NEXT: mov w12, #128 // =0x80
+; CHECK-GI-NEXT: neg x14, x11
+; CHECK-GI-NEXT: lsl x15, x10, x11
+; CHECK-GI-NEXT: sub x12, x12, x11
+; CHECK-GI-NEXT: lsr x14, x10, x14
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x14, x11, #128
-; CHECK-GI-NEXT: lsr x17, x10, x13
-; CHECK-GI-NEXT: csel x18, x16, xzr, lo
-; CHECK-GI-NEXT: sub x12, x12, x14
-; CHECK-GI-NEXT: csel x15, x15, x16, lo
+; CHECK-GI-NEXT: sub x13, x11, #128
+; CHECK-GI-NEXT: lsr x16, x10, x12
+; CHECK-GI-NEXT: csel x18, x15, xzr, lo
+; CHECK-GI-NEXT: lsl x17, x10, x13
+; CHECK-GI-NEXT: csel x14, x14, x15, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: lsl x0, x10, x14
-; CHECK-GI-NEXT: csel x15, xzr, x15, eq
+; CHECK-GI-NEXT: neg x15, x13
+; CHECK-GI-NEXT: csel x14, xzr, x14, eq
+; CHECK-GI-NEXT: cmp x12, #64
+; CHECK-GI-NEXT: lsr x15, x10, x15
+; CHECK-GI-NEXT: csel x16, x16, xzr, lo
+; CHECK-GI-NEXT: cmp x12, #0
+; CHECK-GI-NEXT: csel x12, x10, x16, eq
; CHECK-GI-NEXT: cmp x13, #64
-; CHECK-GI-NEXT: lsr x12, x10, x12
-; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: csel x16, x17, xzr, lo
+; CHECK-GI-NEXT: csel x10, x15, x17, lo
; CHECK-GI-NEXT: cmp x13, #0
-; CHECK-GI-NEXT: csel x13, x10, x17, eq
-; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x10, x12, x0, lo
-; CHECK-GI-NEXT: cmp x14, #0
-; CHECK-GI-NEXT: csel x12, xzr, x10, eq
+; CHECK-GI-NEXT: csel x13, xzr, x10, eq
; CHECK-GI-NEXT: cmp x11, #128
; CHECK-GI-NEXT: csel x10, x18, xzr, lo
-; CHECK-GI-NEXT: csel x15, x15, xzr, lo
-; CHECK-GI-NEXT: csel x13, x13, x16, lo
-; CHECK-GI-NEXT: csel x12, xzr, x12, lo
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: csel x12, x12, x16, lo
+; CHECK-GI-NEXT: csel x13, xzr, x13, lo
; CHECK-GI-NEXT: cmp x11, #0
; CHECK-GI-NEXT: umulh x18, x10, x9
-; CHECK-GI-NEXT: csel x13, xzr, x13, eq
; CHECK-GI-NEXT: csel x12, xzr, x12, eq
-; CHECK-GI-NEXT: umulh x11, x15, x9
-; CHECK-GI-NEXT: umulh x16, x13, x8
-; CHECK-GI-NEXT: umulh x14, x10, x8
+; CHECK-GI-NEXT: csel x13, xzr, x13, eq
+; CHECK-GI-NEXT: umulh x11, x14, x9
+; CHECK-GI-NEXT: umulh x16, x12, x8
+; CHECK-GI-NEXT: umulh x15, x10, x8
; CHECK-GI-NEXT: add x11, x11, x18
; CHECK-GI-NEXT: mul x17, x10, x9
; CHECK-GI-NEXT: add x11, x16, x11
-; CHECK-GI-NEXT: mul x0, x15, x8
-; CHECK-GI-NEXT: umulh x2, x15, x8
-; CHECK-GI-NEXT: adds x14, x14, x17
+; CHECK-GI-NEXT: mul x0, x14, x8
+; CHECK-GI-NEXT: umulh x2, x14, x8
; CHECK-GI-NEXT: madd x11, x10, x9, x11
+; CHECK-GI-NEXT: mul x16, x14, x9
+; CHECK-GI-NEXT: madd x11, x14, x9, x11
+; CHECK-GI-NEXT: adds x14, x15, x17
; CHECK-GI-NEXT: cset w3, hs
; CHECK-GI-NEXT: adds x1, x14, x0
-; CHECK-GI-NEXT: mul x16, x15, x9
+; CHECK-GI-NEXT: mul x15, x12, x8
; CHECK-GI-NEXT: cset w14, hs
; CHECK-GI-NEXT: adds x18, x2, x18
-; CHECK-GI-NEXT: madd x11, x15, x9, x11
; CHECK-GI-NEXT: cset w0, hs
; CHECK-GI-NEXT: adds x17, x18, x17
-; CHECK-GI-NEXT: and x18, x0, #0x1
-; CHECK-GI-NEXT: mul x15, x13, x8
-; CHECK-GI-NEXT: madd x9, x13, x9, x11
+; CHECK-GI-NEXT: madd x9, x12, x9, x11
; CHECK-GI-NEXT: and x11, x3, #0x1
-; CHECK-GI-NEXT: and x13, x14, #0x1
+; CHECK-GI-NEXT: and x12, x14, #0x1
; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: and x18, x0, #0x1
; CHECK-GI-NEXT: and x14, x14, #0x1
; CHECK-GI-NEXT: mul x0, x10, x8
-; CHECK-GI-NEXT: add x10, x11, x13
-; CHECK-GI-NEXT: adds x13, x17, x16
+; CHECK-GI-NEXT: add x10, x11, x12
+; CHECK-GI-NEXT: adds x12, x17, x16
; CHECK-GI-NEXT: add x11, x18, x14
; CHECK-GI-NEXT: cset w14, hs
-; CHECK-GI-NEXT: adds x13, x13, x15
-; CHECK-GI-NEXT: madd x8, x12, x8, x9
+; CHECK-GI-NEXT: adds x12, x12, x15
+; CHECK-GI-NEXT: madd x8, x13, x8, x9
; CHECK-GI-NEXT: and x9, x14, #0x1
-; CHECK-GI-NEXT: cset w12, hs
-; CHECK-GI-NEXT: adds x2, x13, x10
+; CHECK-GI-NEXT: cset w13, hs
+; CHECK-GI-NEXT: adds x2, x12, x10
; CHECK-GI-NEXT: add x9, x11, x9
-; CHECK-GI-NEXT: and x10, x12, #0x1
+; CHECK-GI-NEXT: and x10, x13, #0x1
; CHECK-GI-NEXT: cset w11, hs
; CHECK-GI-NEXT: add x9, x9, x10
; CHECK-GI-NEXT: and x10, x11, #0x1
@@ -1222,34 +1217,33 @@ define i256 @f64_to_u256(double %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB7_3: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub x9, x9, #1075
-; CHECK-GI-NEXT: mov w10, #64 // =0x40
-; CHECK-GI-NEXT: mov w11, #128 // =0x80
-; CHECK-GI-NEXT: sub x12, x10, x9
-; CHECK-GI-NEXT: lsl x13, x8, x9
-; CHECK-GI-NEXT: sub x11, x11, x9
-; CHECK-GI-NEXT: lsr x12, x8, x12
+; CHECK-GI-NEXT: mov w10, #128 // =0x80
+; CHECK-GI-NEXT: neg x11, x9
+; CHECK-GI-NEXT: lsl x12, x8, x9
+; CHECK-GI-NEXT: sub x10, x10, x9
+; CHECK-GI-NEXT: lsr x11, x8, x11
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsr x14, x8, x11
-; CHECK-GI-NEXT: csel x15, x13, xzr, lo
-; CHECK-GI-NEXT: sub x16, x9, #128
-; CHECK-GI-NEXT: csel x12, x12, x13, lo
+; CHECK-GI-NEXT: lsr x13, x8, x10
+; CHECK-GI-NEXT: csel x14, x12, xzr, lo
+; CHECK-GI-NEXT: csel x11, x11, x12, lo
; CHECK-GI-NEXT: cmp x9, #0
-; CHECK-GI-NEXT: sub x10, x10, x16
-; CHECK-GI-NEXT: csel x12, xzr, x12, eq
-; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: lsr x10, x8, x10
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: lsl x11, x8, x16
-; CHECK-GI-NEXT: csel x8, x8, x14, eq
-; CHECK-GI-NEXT: cmp x16, #64
+; CHECK-GI-NEXT: sub x12, x9, #128
+; CHECK-GI-NEXT: csel x11, xzr, x11, eq
+; CHECK-GI-NEXT: cmp x10, #64
+; CHECK-GI-NEXT: neg x15, x12
; CHECK-GI-NEXT: csel x13, x13, xzr, lo
-; CHECK-GI-NEXT: csel x10, x10, x11, lo
-; CHECK-GI-NEXT: cmp x16, #0
+; CHECK-GI-NEXT: cmp x10, #0
+; CHECK-GI-NEXT: lsl x10, x8, x12
+; CHECK-GI-NEXT: lsr x15, x8, x15
+; CHECK-GI-NEXT: csel x8, x8, x13, eq
+; CHECK-GI-NEXT: cmp x12, #64
+; CHECK-GI-NEXT: csel x13, x10, xzr, lo
+; CHECK-GI-NEXT: csel x10, x15, x10, lo
+; CHECK-GI-NEXT: cmp x12, #0
; CHECK-GI-NEXT: csel x10, xzr, x10, eq
; CHECK-GI-NEXT: cmp x9, #128
-; CHECK-GI-NEXT: csel x0, x15, xzr, lo
-; CHECK-GI-NEXT: csel x1, x12, xzr, lo
+; CHECK-GI-NEXT: csel x0, x14, xzr, lo
+; CHECK-GI-NEXT: csel x1, x11, xzr, lo
; CHECK-GI-NEXT: csel x8, x8, x13, lo
; CHECK-GI-NEXT: csel x10, xzr, x10, lo
; CHECK-GI-NEXT: cmp x9, #0
@@ -1434,69 +1428,67 @@ define i256 @f32_to_s256_sat(float %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB8_6: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub w11, w11, #150
-; CHECK-GI-NEXT: mov w13, #64 // =0x40
; CHECK-GI-NEXT: mov w12, #128 // =0x80
-; CHECK-GI-NEXT: sub x15, x13, x11
-; CHECK-GI-NEXT: lsl x16, x10, x11
-; CHECK-GI-NEXT: lsl x17, x10, x11
-; CHECK-GI-NEXT: lsr x15, x10, x15
+; CHECK-GI-NEXT: neg x14, x11
+; CHECK-GI-NEXT: lsl x15, x10, x11
; CHECK-GI-NEXT: sub x12, x12, x11
+; CHECK-GI-NEXT: lsr x14, x10, x14
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x14, x11, #128
-; CHECK-GI-NEXT: lsr x18, x10, x12
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x15, x15, x17, lo
+; CHECK-GI-NEXT: sub x13, x11, #128
+; CHECK-GI-NEXT: lsr x16, x10, x12
+; CHECK-GI-NEXT: csel x18, x15, xzr, lo
+; CHECK-GI-NEXT: lsl x17, x10, x13
+; CHECK-GI-NEXT: csel x14, x14, x15, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: sub x13, x13, x14
-; CHECK-GI-NEXT: csel x15, xzr, x15, eq
+; CHECK-GI-NEXT: neg x15, x13
+; CHECK-GI-NEXT: csel x14, xzr, x14, eq
; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: lsl x0, x10, x14
-; CHECK-GI-NEXT: lsr x13, x10, x13
-; CHECK-GI-NEXT: csel x18, x18, xzr, lo
+; CHECK-GI-NEXT: lsr x15, x10, x15
+; CHECK-GI-NEXT: csel x16, x16, xzr, lo
; CHECK-GI-NEXT: cmp x12, #0
-; CHECK-GI-NEXT: csel x12, x10, x18, eq
-; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: csel x17, x17, xzr, lo
-; CHECK-GI-NEXT: csel x10, x13, x0, lo
-; CHECK-GI-NEXT: cmp x14, #0
+; CHECK-GI-NEXT: csel x12, x10, x16, eq
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x16, x17, xzr, lo
+; CHECK-GI-NEXT: csel x10, x15, x17, lo
+; CHECK-GI-NEXT: cmp x13, #0
; CHECK-GI-NEXT: csel x13, xzr, x10, eq
; CHECK-GI-NEXT: cmp x11, #128
-; CHECK-GI-NEXT: csel x10, x16, xzr, lo
-; CHECK-GI-NEXT: csel x15, x15, xzr, lo
-; CHECK-GI-NEXT: csel x12, x12, x17, lo
+; CHECK-GI-NEXT: csel x10, x18, xzr, lo
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: csel x12, x12, x16, lo
; CHECK-GI-NEXT: csel x13, xzr, x13, lo
; CHECK-GI-NEXT: cmp x11, #0
; CHECK-GI-NEXT: umulh x18, x10, x9
; CHECK-GI-NEXT: csel x12, xzr, x12, eq
; CHECK-GI-NEXT: csel x13, xzr, x13, eq
-; CHECK-GI-NEXT: umulh x11, x15, x9
-; CHECK-GI-NEXT: umulh x17, x12, x8
-; CHECK-GI-NEXT: umulh x14, x10, x8
+; CHECK-GI-NEXT: umulh x11, x14, x9
+; CHECK-GI-NEXT: umulh x16, x12, x8
+; CHECK-GI-NEXT: umulh x15, x10, x8
; CHECK-GI-NEXT: add x11, x11, x18
-; CHECK-GI-NEXT: mul x16, x10, x9
-; CHECK-GI-NEXT: add x11, x17, x11
-; CHECK-GI-NEXT: mul x0, x15, x8
-; CHECK-GI-NEXT: umulh x2, x15, x8
-; CHECK-GI-NEXT: adds x14, x14, x16
+; CHECK-GI-NEXT: mul x17, x10, x9
+; CHECK-GI-NEXT: add x11, x16, x11
+; CHECK-GI-NEXT: mul x0, x14, x8
+; CHECK-GI-NEXT: umulh x2, x14, x8
; CHECK-GI-NEXT: madd x11, x10, x9, x11
+; CHECK-GI-NEXT: mul x16, x14, x9
+; CHECK-GI-NEXT: madd x11, x14, x9, x11
+; CHECK-GI-NEXT: adds x14, x15, x17
; CHECK-GI-NEXT: cset w3, hs
; CHECK-GI-NEXT: adds x1, x14, x0
-; CHECK-GI-NEXT: mul x17, x15, x9
+; CHECK-GI-NEXT: mul x15, x12, x8
; CHECK-GI-NEXT: cset w14, hs
; CHECK-GI-NEXT: adds x18, x2, x18
-; CHECK-GI-NEXT: madd x11, x15, x9, x11
; CHECK-GI-NEXT: cset w0, hs
-; CHECK-GI-NEXT: adds x16, x18, x16
-; CHECK-GI-NEXT: and x18, x0, #0x1
-; CHECK-GI-NEXT: mul x15, x12, x8
+; CHECK-GI-NEXT: adds x17, x18, x17
; CHECK-GI-NEXT: madd x9, x12, x9, x11
; CHECK-GI-NEXT: and x11, x3, #0x1
; CHECK-GI-NEXT: and x12, x14, #0x1
; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: and x18, x0, #0x1
; CHECK-GI-NEXT: and x14, x14, #0x1
; CHECK-GI-NEXT: mul x0, x10, x8
; CHECK-GI-NEXT: add x10, x11, x12
-; CHECK-GI-NEXT: adds x12, x16, x17
+; CHECK-GI-NEXT: adds x12, x17, x16
; CHECK-GI-NEXT: add x11, x18, x14
; CHECK-GI-NEXT: cset w14, hs
; CHECK-GI-NEXT: adds x12, x12, x15
@@ -1640,36 +1632,34 @@ define i256 @f32_to_u256_sat(float %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB9_7: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub w9, w9, #150
-; CHECK-GI-NEXT: mov w11, #64 // =0x40
; CHECK-GI-NEXT: mov w10, #128 // =0x80
-; CHECK-GI-NEXT: sub x12, x11, x9
-; CHECK-GI-NEXT: lsl x13, x8, x9
-; CHECK-GI-NEXT: lsl x14, x8, x9
-; CHECK-GI-NEXT: lsr x12, x8, x12
+; CHECK-GI-NEXT: neg x11, x9
+; CHECK-GI-NEXT: lsl x12, x8, x9
; CHECK-GI-NEXT: sub x10, x10, x9
+; CHECK-GI-NEXT: lsr x11, x8, x11
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsr x15, x8, x10
-; CHECK-GI-NEXT: csel x13, x13, xzr, lo
-; CHECK-GI-NEXT: sub x16, x9, #128
-; CHECK-GI-NEXT: csel x12, x12, x14, lo
+; CHECK-GI-NEXT: lsr x13, x8, x10
+; CHECK-GI-NEXT: csel x14, x12, xzr, lo
+; CHECK-GI-NEXT: csel x11, x11, x12, lo
; CHECK-GI-NEXT: cmp x9, #0
-; CHECK-GI-NEXT: sub x11, x11, x16
-; CHECK-GI-NEXT: csel x12, xzr, x12, eq
+; CHECK-GI-NEXT: sub x12, x9, #128
+; CHECK-GI-NEXT: csel x11, xzr, x11, eq
; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: lsr x11, x8, x11
-; CHECK-GI-NEXT: csel x15, x15, xzr, lo
+; CHECK-GI-NEXT: neg x15, x12
+; CHECK-GI-NEXT: csel x13, x13, xzr, lo
; CHECK-GI-NEXT: cmp x10, #0
-; CHECK-GI-NEXT: lsl x10, x8, x16
-; CHECK-GI-NEXT: csel x8, x8, x15, eq
-; CHECK-GI-NEXT: cmp x16, #64
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: csel x10, x11, x10, lo
-; CHECK-GI-NEXT: cmp x16, #0
+; CHECK-GI-NEXT: lsl x10, x8, x12
+; CHECK-GI-NEXT: lsr x15, x8, x15
+; CHECK-GI-NEXT: csel x8, x8, x13, eq
+; CHECK-GI-NEXT: cmp x12, #64
+; CHECK-GI-NEXT: csel x13, x10, xzr, lo
+; CHECK-GI-NEXT: csel x10, x15, x10, lo
+; CHECK-GI-NEXT: cmp x12, #0
; CHECK-GI-NEXT: csel x10, xzr, x10, eq
; CHECK-GI-NEXT: cmp x9, #128
-; CHECK-GI-NEXT: csel x0, x13, xzr, lo
-; CHECK-GI-NEXT: csel x1, x12, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x14, lo
+; CHECK-GI-NEXT: csel x0, x14, xzr, lo
+; CHECK-GI-NEXT: csel x1, x11, xzr, lo
+; CHECK-GI-NEXT: csel x8, x8, x13, lo
; CHECK-GI-NEXT: csel x10, xzr, x10, lo
; CHECK-GI-NEXT: cmp x9, #0
; CHECK-GI-NEXT: csel x2, xzr, x8, eq
@@ -1855,77 +1845,76 @@ define i256 @f64_to_s256_sat(double %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB10_6: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub x11, x11, #1075
-; CHECK-GI-NEXT: mov w12, #64 // =0x40
-; CHECK-GI-NEXT: mov w13, #128 // =0x80
-; CHECK-GI-NEXT: sub x15, x12, x11
-; CHECK-GI-NEXT: lsl x16, x10, x11
-; CHECK-GI-NEXT: sub x13, x13, x11
-; CHECK-GI-NEXT: lsr x15, x10, x15
+; CHECK-GI-NEXT: mov w12, #128 // =0x80
+; CHECK-GI-NEXT: neg x14, x11
+; CHECK-GI-NEXT: lsl x15, x10, x11
+; CHECK-GI-NEXT: sub x12, x12, x11
+; CHECK-GI-NEXT: lsr x14, x10, x14
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x14, x11, #128
-; CHECK-GI-NEXT: lsr x17, x10, x13
-; CHECK-GI-NEXT: csel x18, x16, xzr, lo
-; CHECK-GI-NEXT: sub x12, x12, x14
-; CHECK-GI-NEXT: csel x15, x15, x16, lo
+; CHECK-GI-NEXT: sub x13, x11, #128
+; CHECK-GI-NEXT: lsr x16, x10, x12
+; CHECK-GI-NEXT: csel x18, x15, xzr, lo
+; CHECK-GI-NEXT: lsl x17, x10, x13
+; CHECK-GI-NEXT: csel x14, x14, x15, lo
; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: lsl x0, x10, x14
-; CHECK-GI-NEXT: csel x15, xzr, x15, eq
+; CHECK-GI-NEXT: neg x15, x13
+; CHECK-GI-NEXT: csel x14, xzr, x14, eq
+; CHECK-GI-NEXT: cmp x12, #64
+; CHECK-GI-NEXT: lsr x15, x10, x15
+; CHECK-GI-NEXT: csel x16, x16, xzr, lo
+; CHECK-GI-NEXT: cmp x12, #0
+; CHECK-GI-NEXT: csel x12, x10, x16, eq
; CHECK-GI-NEXT: cmp x13, #64
-; CHECK-GI-NEXT: lsr x12, x10, x12
-; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: csel x16, x17, xzr, lo
+; CHECK-GI-NEXT: csel x10, x15, x17, lo
; CHECK-GI-NEXT: cmp x13, #0
-; CHECK-GI-NEXT: csel x13, x10, x17, eq
-; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x10, x12, x0, lo
-; CHECK-GI-NEXT: cmp x14, #0
-; CHECK-GI-NEXT: csel x12, xzr, x10, eq
+; CHECK-GI-NEXT: csel x13, xzr, x10, eq
; CHECK-GI-NEXT: cmp x11, #128
; CHECK-GI-NEXT: csel x10, x18, xzr, lo
-; CHECK-GI-NEXT: csel x15, x15, xzr, lo
-; CHECK-GI-NEXT: csel x13, x13, x16, lo
-; CHECK-GI-NEXT: csel x12, xzr, x12, lo
+; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: csel x12, x12, x16, lo
+; CHECK-GI-NEXT: csel x13, xzr, x13, lo
; CHECK-GI-NEXT: cmp x11, #0
; CHECK-GI-NEXT: umulh x18, x10, x9
-; CHECK-GI-NEXT: csel x13, xzr, x13, eq
; CHECK-GI-NEXT: csel x12, xzr, x12, eq
-; CHECK-GI-NEXT: umulh x11, x15, x9
-; CHECK-GI-NEXT: umulh x16, x13, x8
-; CHECK-GI-NEXT: umulh x14, x10, x8
+; CHECK-GI-NEXT: csel x13, xzr, x13, eq
+; CHECK-GI-NEXT: umulh x11, x14, x9
+; CHECK-GI-NEXT: umulh x16, x12, x8
+; CHECK-GI-NEXT: umulh x15, x10, x8
; CHECK-GI-NEXT: add x11, x11, x18
; CHECK-GI-NEXT: mul x17, x10, x9
; CHECK-GI-NEXT: add x11, x16, x11
-; CHECK-GI-NEXT: mul x0, x15, x8
-; CHECK-GI-NEXT: umulh x2, x15, x8
-; CHECK-GI-NEXT: adds x14, x14, x17
+; CHECK-GI-NEXT: mul x0, x14, x8
+; CHECK-GI-NEXT: umulh x2, x14, x8
; CHECK-GI-NEXT: madd x11, x10, x9, x11
+; CHECK-GI-NEXT: mul x16, x14, x9
+; CHECK-GI-NEXT: madd x11, x14, x9, x11
+; CHECK-GI-NEXT: adds x14, x15, x17
; CHECK-GI-NEXT: cset w3, hs
; CHECK-GI-NEXT: adds x1, x14, x0
-; CHECK-GI-NEXT: mul x16, x15, x9
+; CHECK-GI-NEXT: mul x15, x12, x8
; CHECK-GI-NEXT: cset w14, hs
; CHECK-GI-NEXT: adds x18, x2, x18
-; CHECK-GI-NEXT: madd x11, x15, x9, x11
; CHECK-GI-NEXT: cset w0, hs
; CHECK-GI-NEXT: adds x17, x18, x17
-; CHECK-GI-NEXT: and x18, x0, #0x1
-; CHECK-GI-NEXT: mul x15, x13, x8
-; CHECK-GI-NEXT: madd x9, x13, x9, x11
+; CHECK-GI-NEXT: madd x9, x12, x9, x11
; CHECK-GI-NEXT: and x11, x3, #0x1
-; CHECK-GI-NEXT: and x13, x14, #0x1
+; CHECK-GI-NEXT: and x12, x14, #0x1
; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: and x18, x0, #0x1
; CHECK-GI-NEXT: and x14, x14, #0x1
; CHECK-GI-NEXT: mul x0, x10, x8
-; CHECK-GI-NEXT: add x10, x11, x13
-; CHECK-GI-NEXT: adds x13, x17, x16
+; CHECK-GI-NEXT: add x10, x11, x12
+; CHECK-GI-NEXT: adds x12, x17, x16
; CHECK-GI-NEXT: add x11, x18, x14
; CHECK-GI-NEXT: cset w14, hs
-; CHECK-GI-NEXT: adds x13, x13, x15
-; CHECK-GI-NEXT: madd x8, x12, x8, x9
+; CHECK-GI-NEXT: adds x12, x12, x15
+; CHECK-GI-NEXT: madd x8, x13, x8, x9
; CHECK-GI-NEXT: and x9, x14, #0x1
-; CHECK-GI-NEXT: cset w12, hs
-; CHECK-GI-NEXT: adds x2, x13, x10
+; CHECK-GI-NEXT: cset w13, hs
+; CHECK-GI-NEXT: adds x2, x12, x10
; CHECK-GI-NEXT: add x9, x11, x9
-; CHECK-GI-NEXT: and x10, x12, #0x1
+; CHECK-GI-NEXT: and x10, x13, #0x1
; CHECK-GI-NEXT: cset w11, hs
; CHECK-GI-NEXT: add x9, x9, x10
; CHECK-GI-NEXT: and x10, x11, #0x1
@@ -2060,34 +2049,33 @@ define i256 @f64_to_u256_sat(double %val) {
; CHECK-GI-NEXT: ret
; CHECK-GI-NEXT: .LBB11_7: // %fp-to-i-if-exp.large
; CHECK-GI-NEXT: sub x9, x9, #1075
-; CHECK-GI-NEXT: mov w10, #64 // =0x40
-; CHECK-GI-NEXT: mov w11, #128 // =0x80
-; CHECK-GI-NEXT: sub x12, x10, x9
-; CHECK-GI-NEXT: lsl x13, x8, x9
-; CHECK-GI-NEXT: sub x11, x11, x9
-; CHECK-GI-NEXT: lsr x12, x8, x12
+; CHECK-GI-NEXT: mov w10, #128 // =0x80
+; CHECK-GI-NEXT: neg x11, x9
+; CHECK-GI-NEXT: lsl x12, x8, x9
+; CHECK-GI-NEXT: sub x10, x10, x9
+; CHECK-GI-NEXT: lsr x11, x8, x11
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsr x14, x8, x11
-; CHECK-GI-NEXT: csel x15, x13, xzr, lo
-; CHECK-GI-NEXT: sub x16, x9, #128
-; CHECK-GI-NEXT: csel x12, x12, x13, lo
+; CHECK-GI-NEXT: lsr x13, x8, x10
+; CHECK-GI-NEXT: csel x14, x12, xzr, lo
+; CHECK-GI-NEXT: csel x11, x11, x12, lo
; CHECK-GI-NEXT: cmp x9, #0
-; CHECK-GI-NEXT: sub x10, x10, x16
-; CHECK-GI-NEXT: csel x12, xzr, x12, eq
-; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: lsr x10, x8, x10
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: cmp x11, #0
-; CHECK-GI-NEXT: lsl x11, x8, x16
-; CHECK-GI-NEXT: csel x8, x8, x14, eq
-; CHECK-GI-NEXT: cmp x16, #64
+; CHECK-GI-NEXT: sub x12, x9, #128
+; CHECK-GI-NEXT: csel x11, xzr, x11, eq
+; CHECK-GI-NEXT: cmp x10, #64
+; CHECK-GI-NEXT: neg x15, x12
; CHECK-GI-NEXT: csel x13, x13, xzr, lo
-; CHECK-GI-NEXT: csel x10, x10, x11, lo
-; CHECK-GI-NEXT: cmp x16, #0
+; CHECK-GI-NEXT: cmp x10, #0
+; CHECK-GI-NEXT: lsl x10, x8, x12
+; CHECK-GI-NEXT: lsr x15, x8, x15
+; CHECK-GI-NEXT: csel x8, x8, x13, eq
+; CHECK-GI-NEXT: cmp x12, #64
+; CHECK-GI-NEXT: csel x13, x10, xzr, lo
+; CHECK-GI-NEXT: csel x10, x15, x10, lo
+; CHECK-GI-NEXT: cmp x12, #0
; CHECK-GI-NEXT: csel x10, xzr, x10, eq
; CHECK-GI-NEXT: cmp x9, #128
-; CHECK-GI-NEXT: csel x0, x15, xzr, lo
-; CHECK-GI-NEXT: csel x1, x12, xzr, lo
+; CHECK-GI-NEXT: csel x0, x14, xzr, lo
+; CHECK-GI-NEXT: csel x1, x11, xzr, lo
; CHECK-GI-NEXT: csel x8, x8, x13, lo
; CHECK-GI-NEXT: csel x10, xzr, x10, lo
; CHECK-GI-NEXT: cmp x9, #0
diff --git a/llvm/test/CodeGen/AArch64/fsh.ll b/llvm/test/CodeGen/AArch64/fsh.ll
index d174817c7139f..64764be7744e1 100644
--- a/llvm/test/CodeGen/AArch64/fsh.ll
+++ b/llvm/test/CodeGen/AArch64/fsh.ll
@@ -179,35 +179,32 @@ define i128 @rotl_i128(i128 %a, i128 %c) {
;
; CHECK-GI-LABEL: rotl_i128:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: and x9, x2, #0x7f
-; CHECK-GI-NEXT: neg x11, x2
-; CHECK-GI-NEXT: sub x10, x8, x9
-; CHECK-GI-NEXT: lsl x12, x1, x2
-; CHECK-GI-NEXT: lsl x13, x0, x9
-; CHECK-GI-NEXT: lsr x10, x0, x10
-; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: and x9, x11, #0x7f
-; CHECK-GI-NEXT: sub x8, x8, x9
-; CHECK-GI-NEXT: lsr x14, x1, x9
-; CHECK-GI-NEXT: orr x10, x10, x12
-; CHECK-GI-NEXT: lsl x12, x0, x2
+; CHECK-GI-NEXT: and x8, x2, #0x7f
+; CHECK-GI-NEXT: neg x10, x2
+; CHECK-GI-NEXT: neg x9, x8
+; CHECK-GI-NEXT: lsl x11, x1, x8
+; CHECK-GI-NEXT: and x13, x10, #0x7f
+; CHECK-GI-NEXT: lsr x9, x0, x9
+; CHECK-GI-NEXT: lsl x12, x0, x8
+; CHECK-GI-NEXT: cmp x8, #64
+; CHECK-GI-NEXT: neg x8, x13
+; CHECK-GI-NEXT: lsr x14, x1, x13
+; CHECK-GI-NEXT: orr x9, x9, x11
+; CHECK-GI-NEXT: lsr x11, x0, x13
; CHECK-GI-NEXT: lsl x8, x1, x8
-; CHECK-GI-NEXT: csel x10, x10, x13, lo
-; CHECK-GI-NEXT: lsr x13, x0, x11
+; CHECK-GI-NEXT: csel x9, x9, x12, lo
; CHECK-GI-NEXT: csel x12, x12, xzr, lo
; CHECK-GI-NEXT: tst x2, #0x7f
-; CHECK-GI-NEXT: csel x10, x1, x10, eq
-; CHECK-GI-NEXT: orr x8, x13, x8
-; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsr x13, x1, x11
+; CHECK-GI-NEXT: csel x9, x1, x9, eq
+; CHECK-GI-NEXT: orr x8, x11, x8
+; CHECK-GI-NEXT: cmp x13, #64
; CHECK-GI-NEXT: csel x8, x8, x14, lo
-; CHECK-GI-NEXT: tst x11, #0x7f
+; CHECK-GI-NEXT: tst x10, #0x7f
; CHECK-GI-NEXT: csel x8, x0, x8, eq
-; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: csel x9, x13, xzr, lo
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x10, x14, xzr, lo
; CHECK-GI-NEXT: orr x0, x12, x8
-; CHECK-GI-NEXT: orr x1, x10, x9
+; CHECK-GI-NEXT: orr x1, x9, x10
; CHECK-GI-NEXT: ret
entry:
%d = call i128 @llvm.fshl(i128 %a, i128 %a, i128 %c)
@@ -233,34 +230,31 @@ define i128 @rotr_i128(i128 %a, i128 %c) {
;
; CHECK-GI-LABEL: rotr_i128:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: and x9, x2, #0x7f
-; CHECK-GI-NEXT: lsr x11, x0, x2
-; CHECK-GI-NEXT: sub x10, x8, x9
-; CHECK-GI-NEXT: neg x13, x2
-; CHECK-GI-NEXT: lsr x12, x1, x9
-; CHECK-GI-NEXT: lsl x10, x1, x10
-; CHECK-GI-NEXT: cmp x9, #64
+; CHECK-GI-NEXT: and x8, x2, #0x7f
+; CHECK-GI-NEXT: neg x11, x2
+; CHECK-GI-NEXT: neg x9, x8
+; CHECK-GI-NEXT: lsr x10, x0, x8
+; CHECK-GI-NEXT: lsr x12, x1, x8
+; CHECK-GI-NEXT: lsl x9, x1, x9
+; CHECK-GI-NEXT: and x13, x11, #0x7f
+; CHECK-GI-NEXT: cmp x8, #64
; CHECK-GI-NEXT: lsl x14, x1, x13
-; CHECK-GI-NEXT: orr x10, x11, x10
-; CHECK-GI-NEXT: and x11, x13, #0x7f
-; CHECK-GI-NEXT: sub x8, x8, x11
-; CHECK-GI-NEXT: csel x10, x10, x12, lo
-; CHECK-GI-NEXT: lsr x12, x1, x2
+; CHECK-GI-NEXT: orr x9, x10, x9
+; CHECK-GI-NEXT: neg x10, x13
+; CHECK-GI-NEXT: csel x9, x9, x12, lo
; CHECK-GI-NEXT: tst x2, #0x7f
-; CHECK-GI-NEXT: lsr x8, x0, x8
-; CHECK-GI-NEXT: lsl x15, x0, x11
-; CHECK-GI-NEXT: csel x10, x0, x10, eq
-; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsl x9, x0, x13
+; CHECK-GI-NEXT: lsr x10, x0, x10
+; CHECK-GI-NEXT: csel x9, x0, x9, eq
+; CHECK-GI-NEXT: cmp x8, #64
+; CHECK-GI-NEXT: lsl x8, x0, x13
; CHECK-GI-NEXT: csel x12, x12, xzr, lo
-; CHECK-GI-NEXT: orr x8, x8, x14
-; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: csel x9, x9, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x15, lo
-; CHECK-GI-NEXT: tst x13, #0x7f
+; CHECK-GI-NEXT: orr x10, x10, x14
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x13, x8, xzr, lo
+; CHECK-GI-NEXT: csel x8, x10, x8, lo
+; CHECK-GI-NEXT: tst x11, #0x7f
; CHECK-GI-NEXT: csel x8, x1, x8, eq
-; CHECK-GI-NEXT: orr x0, x10, x9
+; CHECK-GI-NEXT: orr x0, x9, x13
; CHECK-GI-NEXT: orr x1, x12, x8
; CHECK-GI-NEXT: ret
entry:
@@ -381,14 +375,25 @@ entry:
}
define i32 @fshl_i32(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: fshl_i32:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: lsr w8, w1, #1
-; CHECK-NEXT: mvn w9, w2
-; CHECK-NEXT: lsl w10, w0, w2
-; CHECK-NEXT: lsr w8, w8, w9
-; CHECK-NEXT: orr w0, w10, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: fshl_i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: lsr w8, w1, #1
+; CHECK-SD-NEXT: mvn w9, w2
+; CHECK-SD-NEXT: lsl w10, w0, w2
+; CHECK-SD-NEXT: lsr w8, w8, w9
+; CHECK-SD-NEXT: orr w0, w10, w8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: fshl_i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: lsr w9, w1, #1
+; CHECK-GI-NEXT: and w10, w2, #0x1f
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsl w10, w0, w10
+; CHECK-GI-NEXT: lsr w8, w9, w8
+; CHECK-GI-NEXT: orr w0, w10, w8
+; CHECK-GI-NEXT: ret
entry:
%d = call i32 @llvm.fshl(i32 %a, i32 %b, i32 %c)
ret i32 %d
@@ -406,10 +411,12 @@ define i32 @fshr_i32(i32 %a, i32 %b, i32 %c) {
;
; CHECK-GI-LABEL: fshr_i32:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: lsl w8, w0, #1
-; CHECK-GI-NEXT: mvn w9, w2
-; CHECK-GI-NEXT: lsl w8, w8, w9
-; CHECK-GI-NEXT: lsr w9, w1, w2
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: lsl w9, w0, #1
+; CHECK-GI-NEXT: and w10, w2, #0x1f
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsl w8, w9, w8
+; CHECK-GI-NEXT: lsr w9, w1, w10
; CHECK-GI-NEXT: orr w0, w8, w9
; CHECK-GI-NEXT: ret
entry:
@@ -429,10 +436,12 @@ define i64 @fshl_i64(i64 %a, i64 %b, i64 %c) {
;
; CHECK-GI-LABEL: fshl_i64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: lsr x8, x1, #1
-; CHECK-GI-NEXT: mvn x9, x2
-; CHECK-GI-NEXT: lsl x10, x0, x2
-; CHECK-GI-NEXT: lsr x8, x8, x9
+; CHECK-GI-NEXT: mov w8, #63 // =0x3f
+; CHECK-GI-NEXT: lsr x9, x1, #1
+; CHECK-GI-NEXT: and x10, x2, #0x3f
+; CHECK-GI-NEXT: bic x8, x8, x2
+; CHECK-GI-NEXT: lsl x10, x0, x10
+; CHECK-GI-NEXT: lsr x8, x9, x8
; CHECK-GI-NEXT: orr x0, x10, x8
; CHECK-GI-NEXT: ret
entry:
@@ -452,10 +461,12 @@ define i64 @fshr_i64(i64 %a, i64 %b, i64 %c) {
;
; CHECK-GI-LABEL: fshr_i64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: lsl x8, x0, #1
-; CHECK-GI-NEXT: mvn x9, x2
-; CHECK-GI-NEXT: lsl x8, x8, x9
-; CHECK-GI-NEXT: lsr x9, x1, x2
+; CHECK-GI-NEXT: mov w8, #63 // =0x3f
+; CHECK-GI-NEXT: lsl x9, x0, #1
+; CHECK-GI-NEXT: and x10, x2, #0x3f
+; CHECK-GI-NEXT: bic x8, x8, x2
+; CHECK-GI-NEXT: lsl x8, x9, x8
+; CHECK-GI-NEXT: lsr x9, x1, x10
; CHECK-GI-NEXT: orr x0, x8, x9
; CHECK-GI-NEXT: ret
entry:
@@ -484,37 +495,34 @@ define i128 @fshl_i128(i128 %a, i128 %b, i128 %c) {
; CHECK-GI-LABEL: fshl_i128:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: and x9, x4, #0x7f
-; CHECK-GI-NEXT: mov w10, #64 // =0x40
; CHECK-GI-NEXT: mov w8, #127 // =0x7f
-; CHECK-GI-NEXT: sub x12, x10, x9
+; CHECK-GI-NEXT: extr x13, x3, x2, #1
+; CHECK-GI-NEXT: neg x11, x9
; CHECK-GI-NEXT: bic x8, x8, x4
-; CHECK-GI-NEXT: lsl x13, x0, x4
-; CHECK-GI-NEXT: lsr x12, x0, x12
-; CHECK-GI-NEXT: lsl x14, x1, x4
-; CHECK-GI-NEXT: extr x16, x3, x2, #1
-; CHECK-GI-NEXT: lsr x17, x3, #1
-; CHECK-GI-NEXT: mvn x11, x4
+; CHECK-GI-NEXT: lsl x12, x1, x9
+; CHECK-GI-NEXT: lsr x11, x0, x11
+; CHECK-GI-NEXT: lsr x14, x3, #1
; CHECK-GI-NEXT: lsl x15, x0, x9
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: sub x9, x10, x8
-; CHECK-GI-NEXT: orr x12, x12, x14
-; CHECK-GI-NEXT: csel x10, x13, xzr, lo
-; CHECK-GI-NEXT: lsr x13, x16, x11
-; CHECK-GI-NEXT: lsl x9, x17, x9
-; CHECK-GI-NEXT: csel x12, x12, x15, lo
+; CHECK-GI-NEXT: neg x9, x8
+; CHECK-GI-NEXT: mvn x10, x4
+; CHECK-GI-NEXT: orr x11, x11, x12
+; CHECK-GI-NEXT: lsr x12, x13, x8
+; CHECK-GI-NEXT: lsl x9, x14, x9
+; CHECK-GI-NEXT: csel x11, x11, x15, lo
+; CHECK-GI-NEXT: csel x15, x15, xzr, lo
; CHECK-GI-NEXT: tst x4, #0x7f
-; CHECK-GI-NEXT: lsr x14, x17, x8
-; CHECK-GI-NEXT: csel x12, x1, x12, eq
-; CHECK-GI-NEXT: orr x9, x13, x9
+; CHECK-GI-NEXT: lsr x14, x14, x8
+; CHECK-GI-NEXT: csel x11, x1, x11, eq
+; CHECK-GI-NEXT: orr x9, x12, x9
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: lsr x13, x17, x11
; CHECK-GI-NEXT: csel x9, x9, x14, lo
-; CHECK-GI-NEXT: tst x11, #0x7f
-; CHECK-GI-NEXT: csel x9, x16, x9, eq
+; CHECK-GI-NEXT: tst x10, #0x7f
+; CHECK-GI-NEXT: csel x9, x13, x9, eq
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: csel x8, x13, xzr, lo
-; CHECK-GI-NEXT: orr x0, x10, x9
-; CHECK-GI-NEXT: orr x1, x12, x8
+; CHECK-GI-NEXT: csel x8, x14, xzr, lo
+; CHECK-GI-NEXT: orr x0, x15, x9
+; CHECK-GI-NEXT: orr x1, x11, x8
; CHECK-GI-NEXT: ret
entry:
%d = call i128 @llvm.fshl(i128 %a, i128 %b, i128 %c)
@@ -545,34 +553,31 @@ define i128 @fshr_i128(i128 %a, i128 %b, i128 %c) {
; CHECK-GI-NEXT: lsl x9, x0, #1
; CHECK-GI-NEXT: extr x10, x1, x0, #63
; CHECK-GI-NEXT: bic x8, x8, x4
-; CHECK-GI-NEXT: mov w11, #64 // =0x40
-; CHECK-GI-NEXT: mvn x12, x4
-; CHECK-GI-NEXT: sub x13, x11, x8
-; CHECK-GI-NEXT: lsl x14, x10, x12
-; CHECK-GI-NEXT: and x15, x4, #0x7f
-; CHECK-GI-NEXT: lsr x13, x9, x13
-; CHECK-GI-NEXT: lsl x16, x9, x12
-; CHECK-GI-NEXT: lsl x9, x9, x8
+; CHECK-GI-NEXT: and x13, x4, #0x7f
+; CHECK-GI-NEXT: mvn x14, x4
+; CHECK-GI-NEXT: neg x11, x8
+; CHECK-GI-NEXT: lsl x12, x10, x8
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: sub x8, x11, x15
-; CHECK-GI-NEXT: orr x13, x13, x14
-; CHECK-GI-NEXT: lsr x14, x2, x4
+; CHECK-GI-NEXT: lsr x11, x9, x11
+; CHECK-GI-NEXT: lsl x9, x9, x8
+; CHECK-GI-NEXT: neg x8, x13
; CHECK-GI-NEXT: lsl x8, x3, x8
-; CHECK-GI-NEXT: csel x11, x16, xzr, lo
-; CHECK-GI-NEXT: csel x9, x13, x9, lo
-; CHECK-GI-NEXT: tst x12, #0x7f
-; CHECK-GI-NEXT: lsr x12, x3, x15
-; CHECK-GI-NEXT: csel x9, x10, x9, eq
-; CHECK-GI-NEXT: orr x8, x14, x8
-; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: lsr x10, x3, x4
-; CHECK-GI-NEXT: csel x8, x8, x12, lo
+; CHECK-GI-NEXT: orr x11, x11, x12
+; CHECK-GI-NEXT: lsr x12, x2, x13
+; CHECK-GI-NEXT: csel x11, x11, x9, lo
+; CHECK-GI-NEXT: csel x9, x9, xzr, lo
+; CHECK-GI-NEXT: tst x14, #0x7f
+; CHECK-GI-NEXT: lsr x14, x3, x13
+; CHECK-GI-NEXT: csel x10, x10, x11, eq
+; CHECK-GI-NEXT: orr x8, x12, x8
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x8, x8, x14, lo
; CHECK-GI-NEXT: tst x4, #0x7f
; CHECK-GI-NEXT: csel x8, x2, x8, eq
-; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: csel x10, x10, xzr, lo
-; CHECK-GI-NEXT: orr x0, x11, x8
-; CHECK-GI-NEXT: orr x1, x9, x10
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: csel x11, x14, xzr, lo
+; CHECK-GI-NEXT: orr x0, x9, x8
+; CHECK-GI-NEXT: orr x1, x10, x11
; CHECK-GI-NEXT: ret
entry:
%d = call i128 @llvm.fshr(i128 %a, i128 %b, i128 %c)
@@ -1736,62 +1741,57 @@ define <2 x i128> @rotl_v2i128(<2 x i128> %a, <2 x i128> %c) {
; CHECK-GI-LABEL: rotl_v2i128:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: and x8, x4, #0x7f
-; CHECK-GI-NEXT: mov w9, #64 // =0x40
-; CHECK-GI-NEXT: lsl x11, x1, x4
-; CHECK-GI-NEXT: sub x10, x9, x8
-; CHECK-GI-NEXT: lsl x13, x0, x4
-; CHECK-GI-NEXT: and x15, x6, #0x7f
-; CHECK-GI-NEXT: lsr x10, x0, x10
-; CHECK-GI-NEXT: lsl x14, x0, x8
+; CHECK-GI-NEXT: and x10, x6, #0x7f
+; CHECK-GI-NEXT: neg x16, x6
+; CHECK-GI-NEXT: neg x9, x8
+; CHECK-GI-NEXT: lsl x11, x1, x8
+; CHECK-GI-NEXT: lsl x12, x0, x8
+; CHECK-GI-NEXT: lsr x9, x0, x9
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: neg x12, x4
-; CHECK-GI-NEXT: lsl x16, x2, x15
-; CHECK-GI-NEXT: neg x17, x6
-; CHECK-GI-NEXT: orr x8, x10, x11
-; CHECK-GI-NEXT: sub x10, x9, x15
-; CHECK-GI-NEXT: csel x11, x13, xzr, lo
-; CHECK-GI-NEXT: lsr x10, x2, x10
-; CHECK-GI-NEXT: lsl x13, x3, x6
-; CHECK-GI-NEXT: csel x8, x8, x14, lo
+; CHECK-GI-NEXT: neg x8, x10
+; CHECK-GI-NEXT: lsl x13, x3, x10
+; CHECK-GI-NEXT: lsr x8, x2, x8
+; CHECK-GI-NEXT: csel x14, x12, xzr, lo
+; CHECK-GI-NEXT: orr x9, x9, x11
+; CHECK-GI-NEXT: neg x11, x4
+; CHECK-GI-NEXT: and x18, x16, #0x7f
+; CHECK-GI-NEXT: csel x9, x9, x12, lo
; CHECK-GI-NEXT: tst x4, #0x7f
-; CHECK-GI-NEXT: lsl x14, x2, x6
-; CHECK-GI-NEXT: and x4, x17, #0x7f
-; CHECK-GI-NEXT: orr x10, x10, x13
-; CHECK-GI-NEXT: and x13, x12, #0x7f
-; CHECK-GI-NEXT: csel x8, x1, x8, eq
-; CHECK-GI-NEXT: cmp x15, #64
-; CHECK-GI-NEXT: sub x15, x9, x13
-; CHECK-GI-NEXT: lsr x18, x1, x13
-; CHECK-GI-NEXT: csel x10, x10, x16, lo
-; CHECK-GI-NEXT: lsr x16, x0, x12
-; CHECK-GI-NEXT: lsl x15, x1, x15
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: lsl x12, x2, x10
+; CHECK-GI-NEXT: orr x8, x8, x13
+; CHECK-GI-NEXT: csel x9, x1, x9, eq
+; CHECK-GI-NEXT: cmp x10, #64
+; CHECK-GI-NEXT: and x10, x11, #0x7f
+; CHECK-GI-NEXT: csel x13, x12, xzr, lo
+; CHECK-GI-NEXT: csel x8, x8, x12, lo
+; CHECK-GI-NEXT: neg x12, x10
+; CHECK-GI-NEXT: lsr x15, x0, x10
; CHECK-GI-NEXT: tst x6, #0x7f
-; CHECK-GI-NEXT: sub x9, x9, x4
-; CHECK-GI-NEXT: csel x10, x3, x10, eq
-; CHECK-GI-NEXT: orr x15, x16, x15
-; CHECK-GI-NEXT: cmp x13, #64
-; CHECK-GI-NEXT: lsr x16, x1, x12
-; CHECK-GI-NEXT: csel x15, x15, x18, lo
-; CHECK-GI-NEXT: tst x12, #0x7f
-; CHECK-GI-NEXT: lsr x12, x2, x17
-; CHECK-GI-NEXT: lsl x9, x3, x9
-; CHECK-GI-NEXT: csel x15, x0, x15, eq
-; CHECK-GI-NEXT: cmp x13, #64
-; CHECK-GI-NEXT: lsr x13, x3, x4
-; CHECK-GI-NEXT: orr x0, x11, x15
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: orr x9, x12, x9
-; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: lsr x12, x3, x17
-; CHECK-GI-NEXT: csel x9, x9, x13, lo
-; CHECK-GI-NEXT: tst x17, #0x7f
-; CHECK-GI-NEXT: csel x9, x2, x9, eq
-; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: orr x1, x8, x16
-; CHECK-GI-NEXT: csel x12, x12, xzr, lo
-; CHECK-GI-NEXT: orr x2, x14, x9
-; CHECK-GI-NEXT: orr x3, x10, x12
+; CHECK-GI-NEXT: lsl x12, x1, x12
+; CHECK-GI-NEXT: lsr x17, x1, x10
+; CHECK-GI-NEXT: csel x8, x3, x8, eq
+; CHECK-GI-NEXT: cmp x10, #64
+; CHECK-GI-NEXT: orr x12, x15, x12
+; CHECK-GI-NEXT: neg x15, x18
+; CHECK-GI-NEXT: csel x12, x12, x17, lo
+; CHECK-GI-NEXT: tst x11, #0x7f
+; CHECK-GI-NEXT: lsr x11, x2, x18
+; CHECK-GI-NEXT: lsl x15, x3, x15
+; CHECK-GI-NEXT: csel x12, x0, x12, eq
+; CHECK-GI-NEXT: cmp x10, #64
+; CHECK-GI-NEXT: lsr x10, x3, x18
+; CHECK-GI-NEXT: csel x17, x17, xzr, lo
+; CHECK-GI-NEXT: cmp x18, #64
+; CHECK-GI-NEXT: orr x11, x11, x15
+; CHECK-GI-NEXT: orr x0, x14, x12
+; CHECK-GI-NEXT: orr x1, x9, x17
+; CHECK-GI-NEXT: csel x11, x11, x10, lo
+; CHECK-GI-NEXT: tst x16, #0x7f
+; CHECK-GI-NEXT: csel x11, x2, x11, eq
+; CHECK-GI-NEXT: cmp x18, #64
+; CHECK-GI-NEXT: csel x10, x10, xzr, lo
+; CHECK-GI-NEXT: orr x2, x13, x11
+; CHECK-GI-NEXT: orr x3, x8, x10
; CHECK-GI-NEXT: ret
entry:
%d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %a, <2 x i128> %c)
@@ -1830,62 +1830,57 @@ define <2 x i128> @rotr_v2i128(<2 x i128> %a, <2 x i128> %c) {
; CHECK-GI-LABEL: rotr_v2i128:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: and x8, x4, #0x7f
-; CHECK-GI-NEXT: mov w9, #64 // =0x40
-; CHECK-GI-NEXT: lsr x13, x0, x4
-; CHECK-GI-NEXT: sub x11, x9, x8
; CHECK-GI-NEXT: and x12, x6, #0x7f
-; CHECK-GI-NEXT: lsr x15, x1, x8
-; CHECK-GI-NEXT: lsl x11, x1, x11
-; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: sub x16, x9, x12
-; CHECK-GI-NEXT: lsr x14, x1, x4
; CHECK-GI-NEXT: neg x10, x4
-; CHECK-GI-NEXT: orr x11, x13, x11
-; CHECK-GI-NEXT: lsr x13, x2, x6
-; CHECK-GI-NEXT: lsl x17, x0, x10
-; CHECK-GI-NEXT: csel x11, x11, x15, lo
+; CHECK-GI-NEXT: neg x9, x8
+; CHECK-GI-NEXT: lsr x11, x0, x8
+; CHECK-GI-NEXT: lsr x13, x1, x8
+; CHECK-GI-NEXT: lsl x9, x1, x9
+; CHECK-GI-NEXT: cmp x8, #64
+; CHECK-GI-NEXT: neg x15, x12
+; CHECK-GI-NEXT: lsr x14, x2, x12
+; CHECK-GI-NEXT: orr x9, x11, x9
+; CHECK-GI-NEXT: lsl x11, x3, x15
+; CHECK-GI-NEXT: lsr x15, x3, x12
+; CHECK-GI-NEXT: csel x9, x9, x13, lo
; CHECK-GI-NEXT: tst x4, #0x7f
-; CHECK-GI-NEXT: lsl x15, x3, x16
-; CHECK-GI-NEXT: csel x11, x0, x11, eq
+; CHECK-GI-NEXT: csel x9, x0, x9, eq
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: lsr x8, x3, x12
-; CHECK-GI-NEXT: orr x13, x13, x15
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
+; CHECK-GI-NEXT: orr x8, x14, x11
+; CHECK-GI-NEXT: csel x11, x13, xzr, lo
; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: csel x8, x13, x8, lo
+; CHECK-GI-NEXT: and x13, x10, #0x7f
+; CHECK-GI-NEXT: csel x8, x8, x15, lo
; CHECK-GI-NEXT: tst x6, #0x7f
-; CHECK-GI-NEXT: lsr x15, x3, x6
+; CHECK-GI-NEXT: neg x14, x6
+; CHECK-GI-NEXT: neg x16, x13
; CHECK-GI-NEXT: csel x8, x2, x8, eq
; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: and x12, x10, #0x7f
-; CHECK-GI-NEXT: neg x13, x6
-; CHECK-GI-NEXT: sub x16, x9, x12
-; CHECK-GI-NEXT: lsl x18, x1, x10
-; CHECK-GI-NEXT: lsr x16, x0, x16
-; CHECK-GI-NEXT: and x4, x13, #0x7f
+; CHECK-GI-NEXT: lsr x12, x0, x16
+; CHECK-GI-NEXT: lsl x16, x1, x13
+; CHECK-GI-NEXT: and x17, x14, #0x7f
; CHECK-GI-NEXT: csel x15, x15, xzr, lo
-; CHECK-GI-NEXT: lsl x0, x0, x12
-; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: sub x9, x9, x4
-; CHECK-GI-NEXT: orr x12, x16, x18
-; CHECK-GI-NEXT: csel x16, x17, xzr, lo
-; CHECK-GI-NEXT: lsr x9, x2, x9
-; CHECK-GI-NEXT: lsl x17, x3, x13
-; CHECK-GI-NEXT: csel x12, x12, x0, lo
+; CHECK-GI-NEXT: lsl x18, x0, x13
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: neg x13, x17
+; CHECK-GI-NEXT: orr x12, x12, x16
+; CHECK-GI-NEXT: lsl x16, x3, x17
+; CHECK-GI-NEXT: lsr x13, x2, x13
+; CHECK-GI-NEXT: csel x12, x12, x18, lo
+; CHECK-GI-NEXT: csel x18, x18, xzr, lo
; CHECK-GI-NEXT: tst x10, #0x7f
-; CHECK-GI-NEXT: lsl x10, x2, x13
-; CHECK-GI-NEXT: lsl x18, x2, x4
+; CHECK-GI-NEXT: lsl x10, x2, x17
+; CHECK-GI-NEXT: orr x0, x9, x18
; CHECK-GI-NEXT: csel x12, x1, x12, eq
-; CHECK-GI-NEXT: orr x9, x9, x17
-; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: orr x0, x11, x16
-; CHECK-GI-NEXT: csel x10, x10, xzr, lo
-; CHECK-GI-NEXT: csel x9, x9, x18, lo
-; CHECK-GI-NEXT: tst x13, #0x7f
-; CHECK-GI-NEXT: csel x9, x3, x9, eq
-; CHECK-GI-NEXT: orr x1, x14, x12
-; CHECK-GI-NEXT: orr x2, x8, x10
-; CHECK-GI-NEXT: orr x3, x15, x9
+; CHECK-GI-NEXT: orr x13, x13, x16
+; CHECK-GI-NEXT: cmp x17, #64
+; CHECK-GI-NEXT: csel x16, x10, xzr, lo
+; CHECK-GI-NEXT: csel x10, x13, x10, lo
+; CHECK-GI-NEXT: tst x14, #0x7f
+; CHECK-GI-NEXT: csel x10, x3, x10, eq
+; CHECK-GI-NEXT: orr x1, x11, x12
+; CHECK-GI-NEXT: orr x2, x8, x16
+; CHECK-GI-NEXT: orr x3, x15, x10
; CHECK-GI-NEXT: ret
entry:
%d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %a, <2 x i128> %c)
@@ -2829,70 +2824,65 @@ define <2 x i128> @fshl_v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
;
; CHECK-GI-LABEL: fshl_v2i128:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldr x8, [sp]
-; CHECK-GI-NEXT: mov w11, #64 // =0x40
-; CHECK-GI-NEXT: ldr x9, [sp, #16]
-; CHECK-GI-NEXT: mov w12, #127 // =0x7f
-; CHECK-GI-NEXT: and x10, x8, #0x7f
-; CHECK-GI-NEXT: lsl x15, x1, x8
-; CHECK-GI-NEXT: and x14, x9, #0x7f
-; CHECK-GI-NEXT: sub x13, x11, x10
-; CHECK-GI-NEXT: lsl x17, x0, x8
-; CHECK-GI-NEXT: cmp x10, #64
-; CHECK-GI-NEXT: lsr x13, x0, x13
-; CHECK-GI-NEXT: lsl x18, x0, x10
-; CHECK-GI-NEXT: mvn x16, x8
-; CHECK-GI-NEXT: csel x10, x17, xzr, lo
-; CHECK-GI-NEXT: lsl x17, x3, x9
-; CHECK-GI-NEXT: mvn x0, x9
-; CHECK-GI-NEXT: orr x13, x13, x15
-; CHECK-GI-NEXT: sub x15, x11, x14
-; CHECK-GI-NEXT: lsr x15, x2, x15
-; CHECK-GI-NEXT: csel x13, x13, x18, lo
-; CHECK-GI-NEXT: tst x8, #0x7f
-; CHECK-GI-NEXT: bic x8, x12, x8
-; CHECK-GI-NEXT: csel x13, x1, x13, eq
-; CHECK-GI-NEXT: lsl x18, x2, x9
-; CHECK-GI-NEXT: orr x15, x15, x17
-; CHECK-GI-NEXT: lsl x17, x2, x14
-; CHECK-GI-NEXT: cmp x14, #64
-; CHECK-GI-NEXT: lsr x14, x5, #1
-; CHECK-GI-NEXT: extr x1, x5, x4, #1
-; CHECK-GI-NEXT: bic x12, x12, x9
-; CHECK-GI-NEXT: csel x15, x15, x17, lo
-; CHECK-GI-NEXT: sub x17, x11, x8
-; CHECK-GI-NEXT: csel x18, x18, xzr, lo
+; CHECK-GI-NEXT: ldr x9, [sp]
+; CHECK-GI-NEXT: ldr x10, [sp, #16]
+; CHECK-GI-NEXT: mov w8, #127 // =0x7f
+; CHECK-GI-NEXT: and x11, x9, #0x7f
+; CHECK-GI-NEXT: and x13, x10, #0x7f
+; CHECK-GI-NEXT: mvn x15, x9
+; CHECK-GI-NEXT: neg x12, x11
+; CHECK-GI-NEXT: lsl x14, x1, x11
+; CHECK-GI-NEXT: lsl x17, x0, x11
+; CHECK-GI-NEXT: lsr x12, x0, x12
+; CHECK-GI-NEXT: cmp x11, #64
+; CHECK-GI-NEXT: bic x16, x8, x9
+; CHECK-GI-NEXT: csel x18, x17, xzr, lo
+; CHECK-GI-NEXT: bic x8, x8, x10
+; CHECK-GI-NEXT: orr x12, x12, x14
+; CHECK-GI-NEXT: neg x14, x13
+; CHECK-GI-NEXT: lsr x11, x2, x14
+; CHECK-GI-NEXT: lsl x14, x3, x13
+; CHECK-GI-NEXT: csel x12, x12, x17, lo
; CHECK-GI-NEXT: tst x9, #0x7f
-; CHECK-GI-NEXT: lsr x9, x1, x16
-; CHECK-GI-NEXT: lsl x17, x14, x17
-; CHECK-GI-NEXT: csel x15, x3, x15, eq
-; CHECK-GI-NEXT: lsr x2, x7, #1
-; CHECK-GI-NEXT: lsr x3, x14, x8
-; CHECK-GI-NEXT: extr x4, x7, x6, #1
-; CHECK-GI-NEXT: orr x9, x9, x17
+; CHECK-GI-NEXT: csel x9, x1, x12, eq
+; CHECK-GI-NEXT: lsl x12, x2, x13
+; CHECK-GI-NEXT: orr x11, x11, x14
+; CHECK-GI-NEXT: cmp x13, #64
+; CHECK-GI-NEXT: lsr x13, x5, #1
+; CHECK-GI-NEXT: extr x14, x5, x4, #1
+; CHECK-GI-NEXT: csel x17, x12, xzr, lo
+; CHECK-GI-NEXT: csel x11, x11, x12, lo
+; CHECK-GI-NEXT: neg x12, x16
+; CHECK-GI-NEXT: lsr x0, x14, x16
+; CHECK-GI-NEXT: lsl x12, x13, x12
+; CHECK-GI-NEXT: tst x10, #0x7f
+; CHECK-GI-NEXT: lsr x1, x7, #1
+; CHECK-GI-NEXT: lsr x13, x13, x16
+; CHECK-GI-NEXT: extr x2, x7, x6, #1
+; CHECK-GI-NEXT: csel x11, x3, x11, eq
+; CHECK-GI-NEXT: orr x12, x0, x12
+; CHECK-GI-NEXT: cmp x16, #64
+; CHECK-GI-NEXT: neg x0, x8
+; CHECK-GI-NEXT: csel x12, x12, x13, lo
+; CHECK-GI-NEXT: tst x15, #0x7f
+; CHECK-GI-NEXT: lsr x15, x2, x8
+; CHECK-GI-NEXT: lsl x0, x1, x0
+; CHECK-GI-NEXT: csel x12, x14, x12, eq
+; CHECK-GI-NEXT: cmp x16, #64
+; CHECK-GI-NEXT: lsr x14, x1, x8
+; CHECK-GI-NEXT: mvn x10, x10
+; CHECK-GI-NEXT: csel x13, x13, xzr, lo
+; CHECK-GI-NEXT: orr x15, x15, x0
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: sub x11, x11, x12
-; CHECK-GI-NEXT: lsr x14, x14, x16
-; CHECK-GI-NEXT: csel x9, x9, x3, lo
-; CHECK-GI-NEXT: tst x16, #0x7f
-; CHECK-GI-NEXT: lsr x16, x4, x0
-; CHECK-GI-NEXT: lsl x11, x2, x11
-; CHECK-GI-NEXT: csel x9, x1, x9, eq
+; CHECK-GI-NEXT: csel x15, x15, x14, lo
+; CHECK-GI-NEXT: tst x10, #0x7f
+; CHECK-GI-NEXT: orr x0, x18, x12
+; CHECK-GI-NEXT: csel x10, x2, x15, eq
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: lsr x8, x2, x12
-; CHECK-GI-NEXT: csel x14, x14, xzr, lo
-; CHECK-GI-NEXT: orr x11, x16, x11
-; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: lsr x16, x2, x0
-; CHECK-GI-NEXT: csel x8, x11, x8, lo
-; CHECK-GI-NEXT: tst x0, #0x7f
-; CHECK-GI-NEXT: csel x8, x4, x8, eq
-; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: orr x0, x10, x9
-; CHECK-GI-NEXT: csel x11, x16, xzr, lo
-; CHECK-GI-NEXT: orr x1, x13, x14
-; CHECK-GI-NEXT: orr x2, x18, x8
-; CHECK-GI-NEXT: orr x3, x15, x11
+; CHECK-GI-NEXT: orr x1, x9, x13
+; CHECK-GI-NEXT: csel x8, x14, xzr, lo
+; CHECK-GI-NEXT: orr x2, x17, x10
+; CHECK-GI-NEXT: orr x3, x11, x8
; CHECK-GI-NEXT: ret
entry:
%d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
@@ -2934,74 +2924,65 @@ define <2 x i128> @fshr_v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
;
; CHECK-GI-LABEL: fshr_v2i128:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: str x19, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: .cfi_offset w19, -16
-; CHECK-GI-NEXT: ldr x10, [sp, #16]
-; CHECK-GI-NEXT: mov w12, #127 // =0x7f
-; CHECK-GI-NEXT: mov w14, #64 // =0x40
-; CHECK-GI-NEXT: lsl x15, x0, #1
-; CHECK-GI-NEXT: extr x16, x1, x0, #63
-; CHECK-GI-NEXT: ldr x8, [sp, #32]
-; CHECK-GI-NEXT: bic x13, x12, x10
-; CHECK-GI-NEXT: mvn x17, x10
-; CHECK-GI-NEXT: lsl x19, x2, #1
-; CHECK-GI-NEXT: sub x0, x14, x13
-; CHECK-GI-NEXT: lsl x1, x16, x17
-; CHECK-GI-NEXT: bic x12, x12, x8
-; CHECK-GI-NEXT: lsr x0, x15, x0
-; CHECK-GI-NEXT: extr x2, x3, x2, #63
-; CHECK-GI-NEXT: lsl x3, x15, x17
-; CHECK-GI-NEXT: lsl x15, x15, x13
+; CHECK-GI-NEXT: ldr x9, [sp]
+; CHECK-GI-NEXT: mov w11, #127 // =0x7f
+; CHECK-GI-NEXT: lsl x13, x0, #1
+; CHECK-GI-NEXT: extr x14, x1, x0, #63
+; CHECK-GI-NEXT: ldr x8, [sp, #16]
+; CHECK-GI-NEXT: lsl x0, x2, #1
+; CHECK-GI-NEXT: bic x12, x11, x9
+; CHECK-GI-NEXT: extr x1, x3, x2, #63
+; CHECK-GI-NEXT: mvn x17, x9
+; CHECK-GI-NEXT: neg x16, x12
+; CHECK-GI-NEXT: lsl x18, x14, x12
+; CHECK-GI-NEXT: bic x11, x11, x8
+; CHECK-GI-NEXT: lsr x16, x13, x16
+; CHECK-GI-NEXT: lsl x13, x13, x12
+; CHECK-GI-NEXT: cmp x12, #64
+; CHECK-GI-NEXT: and x15, x9, #0x7f
+; CHECK-GI-NEXT: and x10, x8, #0x7f
+; CHECK-GI-NEXT: orr x16, x16, x18
+; CHECK-GI-NEXT: csel x12, x13, xzr, lo
; CHECK-GI-NEXT: mvn x18, x8
-; CHECK-GI-NEXT: cmp x13, #64
-; CHECK-GI-NEXT: orr x0, x0, x1
-; CHECK-GI-NEXT: sub x1, x14, x12
-; CHECK-GI-NEXT: and x11, x10, #0x7f
-; CHECK-GI-NEXT: lsr x13, x19, x1
-; CHECK-GI-NEXT: lsl x1, x2, x18
-; CHECK-GI-NEXT: csel x3, x3, xzr, lo
-; CHECK-GI-NEXT: csel x15, x0, x15, lo
+; CHECK-GI-NEXT: csel x13, x16, x13, lo
+; CHECK-GI-NEXT: neg x16, x11
; CHECK-GI-NEXT: tst x17, #0x7f
-; CHECK-GI-NEXT: lsl x17, x19, x12
-; CHECK-GI-NEXT: csel x15, x16, x15, eq
-; CHECK-GI-NEXT: lsl x16, x19, x18
-; CHECK-GI-NEXT: cmp x12, #64
-; CHECK-GI-NEXT: sub x12, x14, x11
-; CHECK-GI-NEXT: orr x13, x13, x1
-; CHECK-GI-NEXT: lsr x0, x4, x10
-; CHECK-GI-NEXT: lsl x12, x5, x12
-; CHECK-GI-NEXT: and x9, x8, #0x7f
-; CHECK-GI-NEXT: csel x16, x16, xzr, lo
-; CHECK-GI-NEXT: csel x13, x13, x17, lo
-; CHECK-GI-NEXT: tst x18, #0x7f
-; CHECK-GI-NEXT: lsr x17, x5, x11
-; CHECK-GI-NEXT: csel x13, x2, x13, eq
-; CHECK-GI-NEXT: orr x12, x0, x12
-; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: sub x14, x14, x9
-; CHECK-GI-NEXT: lsr x18, x5, x10
-; CHECK-GI-NEXT: csel x12, x12, x17, lo
-; CHECK-GI-NEXT: tst x10, #0x7f
-; CHECK-GI-NEXT: lsr x10, x6, x8
-; CHECK-GI-NEXT: lsl x14, x7, x14
-; CHECK-GI-NEXT: csel x12, x4, x12, eq
+; CHECK-GI-NEXT: lsr x16, x0, x16
+; CHECK-GI-NEXT: lsl x17, x1, x11
+; CHECK-GI-NEXT: csel x13, x14, x13, eq
+; CHECK-GI-NEXT: lsl x14, x0, x11
; CHECK-GI-NEXT: cmp x11, #64
-; CHECK-GI-NEXT: lsr x11, x7, x9
-; CHECK-GI-NEXT: csel x17, x18, xzr, lo
-; CHECK-GI-NEXT: orr x10, x10, x14
-; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: lsr x14, x7, x8
-; CHECK-GI-NEXT: csel x10, x10, x11, lo
+; CHECK-GI-NEXT: neg x11, x15
+; CHECK-GI-NEXT: orr x16, x16, x17
+; CHECK-GI-NEXT: lsr x0, x4, x15
+; CHECK-GI-NEXT: lsl x11, x5, x11
+; CHECK-GI-NEXT: csel x17, x14, xzr, lo
+; CHECK-GI-NEXT: csel x14, x16, x14, lo
+; CHECK-GI-NEXT: tst x18, #0x7f
+; CHECK-GI-NEXT: lsr x16, x5, x15
+; CHECK-GI-NEXT: csel x14, x1, x14, eq
+; CHECK-GI-NEXT: orr x11, x0, x11
+; CHECK-GI-NEXT: cmp x15, #64
+; CHECK-GI-NEXT: neg x18, x10
+; CHECK-GI-NEXT: csel x11, x11, x16, lo
+; CHECK-GI-NEXT: tst x9, #0x7f
+; CHECK-GI-NEXT: lsr x9, x6, x10
+; CHECK-GI-NEXT: lsl x18, x7, x18
+; CHECK-GI-NEXT: csel x11, x4, x11, eq
+; CHECK-GI-NEXT: cmp x15, #64
+; CHECK-GI-NEXT: lsr x15, x7, x10
+; CHECK-GI-NEXT: csel x16, x16, xzr, lo
+; CHECK-GI-NEXT: cmp x10, #64
+; CHECK-GI-NEXT: orr x9, x9, x18
+; CHECK-GI-NEXT: orr x0, x12, x11
+; CHECK-GI-NEXT: orr x1, x13, x16
+; CHECK-GI-NEXT: csel x9, x9, x15, lo
; CHECK-GI-NEXT: tst x8, #0x7f
-; CHECK-GI-NEXT: csel x8, x6, x10, eq
-; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: orr x0, x3, x12
-; CHECK-GI-NEXT: csel x9, x14, xzr, lo
-; CHECK-GI-NEXT: orr x1, x15, x17
-; CHECK-GI-NEXT: orr x2, x16, x8
-; CHECK-GI-NEXT: orr x3, x13, x9
-; CHECK-GI-NEXT: ldr x19, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT: csel x8, x6, x9, eq
+; CHECK-GI-NEXT: cmp x10, #64
+; CHECK-GI-NEXT: csel x9, x15, xzr, lo
+; CHECK-GI-NEXT: orr x2, x17, x8
+; CHECK-GI-NEXT: orr x3, x14, x9
; CHECK-GI-NEXT: ret
entry:
%d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
diff --git a/llvm/test/CodeGen/AArch64/funnel-shift.ll b/llvm/test/CodeGen/AArch64/funnel-shift.ll
index d750e9521096d..ee2faf9ad747e 100644
--- a/llvm/test/CodeGen/AArch64/funnel-shift.ll
+++ b/llvm/test/CodeGen/AArch64/funnel-shift.ll
@@ -18,14 +18,25 @@ declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
; General case - all operands can be variables.
define i32 @fshl_i32(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: fshl_i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: lsr w8, w1, #1
-; CHECK-NEXT: mvn w9, w2
-; CHECK-NEXT: lsl w10, w0, w2
-; CHECK-NEXT: lsr w8, w8, w9
-; CHECK-NEXT: orr w0, w10, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: fshl_i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: lsr w8, w1, #1
+; CHECK-SD-NEXT: mvn w9, w2
+; CHECK-SD-NEXT: lsl w10, w0, w2
+; CHECK-SD-NEXT: lsr w8, w8, w9
+; CHECK-SD-NEXT: orr w0, w10, w8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: fshl_i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: lsr w9, w1, #1
+; CHECK-GI-NEXT: and w10, w2, #0x1f
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsl w10, w0, w10
+; CHECK-GI-NEXT: lsr w8, w9, w8
+; CHECK-GI-NEXT: orr w0, w10, w8
+; CHECK-GI-NEXT: ret
%f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z)
ret i32 %f
}
@@ -42,10 +53,12 @@ define i64 @fshl_i64(i64 %x, i64 %y, i64 %z) {
;
; CHECK-GI-LABEL: fshl_i64:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: lsr x8, x1, #1
-; CHECK-GI-NEXT: mvn x9, x2
-; CHECK-GI-NEXT: lsl x10, x0, x2
-; CHECK-GI-NEXT: lsr x8, x8, x9
+; CHECK-GI-NEXT: mov w8, #63 // =0x3f
+; CHECK-GI-NEXT: lsr x9, x1, #1
+; CHECK-GI-NEXT: and x10, x2, #0x3f
+; CHECK-GI-NEXT: bic x8, x8, x2
+; CHECK-GI-NEXT: lsl x10, x0, x10
+; CHECK-GI-NEXT: lsr x8, x9, x8
; CHECK-GI-NEXT: orr x0, x10, x8
; CHECK-GI-NEXT: ret
%f = call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 %z)
@@ -73,37 +86,34 @@ define i128 @fshl_i128(i128 %x, i128 %y, i128 %z) nounwind {
; CHECK-GI-LABEL: fshl_i128:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: and x9, x4, #0x7f
-; CHECK-GI-NEXT: mov w10, #64 // =0x40
; CHECK-GI-NEXT: mov w8, #127 // =0x7f
-; CHECK-GI-NEXT: sub x12, x10, x9
+; CHECK-GI-NEXT: extr x13, x3, x2, #1
+; CHECK-GI-NEXT: neg x11, x9
; CHECK-GI-NEXT: bic x8, x8, x4
-; CHECK-GI-NEXT: lsl x13, x0, x4
-; CHECK-GI-NEXT: lsr x12, x0, x12
-; CHECK-GI-NEXT: lsl x14, x1, x4
-; CHECK-GI-NEXT: extr x16, x3, x2, #1
-; CHECK-GI-NEXT: lsr x17, x3, #1
-; CHECK-GI-NEXT: mvn x11, x4
+; CHECK-GI-NEXT: lsl x12, x1, x9
+; CHECK-GI-NEXT: lsr x11, x0, x11
+; CHECK-GI-NEXT: lsr x14, x3, #1
; CHECK-GI-NEXT: lsl x15, x0, x9
; CHECK-GI-NEXT: cmp x9, #64
-; CHECK-GI-NEXT: sub x9, x10, x8
-; CHECK-GI-NEXT: orr x12, x12, x14
-; CHECK-GI-NEXT: csel x10, x13, xzr, lo
-; CHECK-GI-NEXT: lsr x13, x16, x11
-; CHECK-GI-NEXT: lsl x9, x17, x9
-; CHECK-GI-NEXT: csel x12, x12, x15, lo
+; CHECK-GI-NEXT: neg x9, x8
+; CHECK-GI-NEXT: mvn x10, x4
+; CHECK-GI-NEXT: orr x11, x11, x12
+; CHECK-GI-NEXT: lsr x12, x13, x8
+; CHECK-GI-NEXT: lsl x9, x14, x9
+; CHECK-GI-NEXT: csel x11, x11, x15, lo
+; CHECK-GI-NEXT: csel x15, x15, xzr, lo
; CHECK-GI-NEXT: tst x4, #0x7f
-; CHECK-GI-NEXT: lsr x14, x17, x8
-; CHECK-GI-NEXT: csel x12, x1, x12, eq
-; CHECK-GI-NEXT: orr x9, x13, x9
+; CHECK-GI-NEXT: lsr x14, x14, x8
+; CHECK-GI-NEXT: csel x11, x1, x11, eq
+; CHECK-GI-NEXT: orr x9, x12, x9
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: lsr x13, x17, x11
; CHECK-GI-NEXT: csel x9, x9, x14, lo
-; CHECK-GI-NEXT: tst x11, #0x7f
-; CHECK-GI-NEXT: csel x9, x16, x9, eq
+; CHECK-GI-NEXT: tst x10, #0x7f
+; CHECK-GI-NEXT: csel x9, x13, x9, eq
; CHECK-GI-NEXT: cmp x8, #64
-; CHECK-GI-NEXT: csel x8, x13, xzr, lo
-; CHECK-GI-NEXT: orr x0, x10, x9
-; CHECK-GI-NEXT: orr x1, x12, x8
+; CHECK-GI-NEXT: csel x8, x14, xzr, lo
+; CHECK-GI-NEXT: orr x0, x15, x9
+; CHECK-GI-NEXT: orr x1, x11, x8
; CHECK-GI-NEXT: ret
%f = call i128 @llvm.fshl.i128(i128 %x, i128 %y, i128 %z)
ret i128 %f
@@ -138,6 +148,8 @@ define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) {
; CHECK-GI-NEXT: mov w9, #36 // =0x24
; CHECK-GI-NEXT: ubfx x10, x1, #1, #36
; CHECK-GI-NEXT: sub x9, x9, x8
+; CHECK-GI-NEXT: and x8, x8, #0x1fffffffff
+; CHECK-GI-NEXT: and x9, x9, #0x1fffffffff
; CHECK-GI-NEXT: lsl x8, x0, x8
; CHECK-GI-NEXT: lsr x9, x10, x9
; CHECK-GI-NEXT: orr x0, x8, x9
@@ -283,10 +295,12 @@ define i32 @fshr_i32(i32 %x, i32 %y, i32 %z) {
;
; CHECK-GI-LABEL: fshr_i32:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: lsl w8, w0, #1
-; CHECK-GI-NEXT: mvn w9, w2
-; CHECK-GI-NEXT: lsl w8, w8, w9
-; CHECK-GI-NEXT: lsr w9, w1, w2
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: lsl w9, w0, #1
+; CHECK-GI-NEXT: and w10, w2, #0x1f
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsl w8, w9, w8
+; CHECK-GI-NEXT: lsr w9, w1, w10
; CHECK-GI-NEXT: orr w0, w8, w9
; CHECK-GI-NEXT: ret
%f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
@@ -305,10 +319,12 @@ define i64 @fshr_i64(i64 %x, i64 %y, i64 %z) {
;
; CHECK-GI-LABEL: fshr_i64:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: lsl x8, x0, #1
-; CHECK-GI-NEXT: mvn x9, x2
-; CHECK-GI-NEXT: lsl x8, x8, x9
-; CHECK-GI-NEXT: lsr x9, x1, x2
+; CHECK-GI-NEXT: mov w8, #63 // =0x3f
+; CHECK-GI-NEXT: lsl x9, x0, #1
+; CHECK-GI-NEXT: and x10, x2, #0x3f
+; CHECK-GI-NEXT: bic x8, x8, x2
+; CHECK-GI-NEXT: lsl x8, x9, x8
+; CHECK-GI-NEXT: lsr x9, x1, x10
; CHECK-GI-NEXT: orr x0, x8, x9
; CHECK-GI-NEXT: ret
%f = call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 %z)
@@ -347,6 +363,8 @@ define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) {
; CHECK-GI-NEXT: mov w9, #36 // =0x24
; CHECK-GI-NEXT: lsl x10, x0, #1
; CHECK-GI-NEXT: sub x9, x9, x8
+; CHECK-GI-NEXT: and x8, x8, #0x1fffffffff
+; CHECK-GI-NEXT: and x9, x9, #0x1fffffffff
; CHECK-GI-NEXT: lsr x8, x11, x8
; CHECK-GI-NEXT: lsl x9, x10, x9
; CHECK-GI-NEXT: orr x0, x9, x8
@@ -528,12 +546,14 @@ define i32 @or_shl_fshl(i32 %x, i32 %y, i32 %s) {
;
; CHECK-GI-LABEL: or_shl_fshl:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: lsr w8, w1, #1
-; CHECK-GI-NEXT: lsl w9, w1, w2
-; CHECK-GI-NEXT: mvn w10, w2
-; CHECK-GI-NEXT: lsl w11, w0, w2
-; CHECK-GI-NEXT: lsr w8, w8, w10
-; CHECK-GI-NEXT: orr w9, w11, w9
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: and w9, w2, #0x1f
+; CHECK-GI-NEXT: lsr w10, w1, #1
+; CHECK-GI-NEXT: lsl w11, w1, w2
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsl w9, w0, w9
+; CHECK-GI-NEXT: lsr w8, w10, w8
+; CHECK-GI-NEXT: orr w9, w9, w11
; CHECK-GI-NEXT: orr w0, w9, w8
; CHECK-GI-NEXT: ret
%shy = shl i32 %y, %s
@@ -569,12 +589,14 @@ define i32 @or_shl_fshl_commute(i32 %x, i32 %y, i32 %s) {
;
; CHECK-GI-LABEL: or_shl_fshl_commute:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: lsr w8, w1, #1
-; CHECK-GI-NEXT: lsl w9, w1, w2
-; CHECK-GI-NEXT: mvn w10, w2
-; CHECK-GI-NEXT: lsl w11, w0, w2
-; CHECK-GI-NEXT: lsr w8, w8, w10
-; CHECK-GI-NEXT: orr w9, w9, w11
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: and w9, w2, #0x1f
+; CHECK-GI-NEXT: lsr w10, w1, #1
+; CHECK-GI-NEXT: lsl w11, w1, w2
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsl w9, w0, w9
+; CHECK-GI-NEXT: lsr w8, w10, w8
+; CHECK-GI-NEXT: orr w9, w11, w9
; CHECK-GI-NEXT: orr w0, w9, w8
; CHECK-GI-NEXT: ret
%shy = shl i32 %y, %s
@@ -610,12 +632,14 @@ define i32 @or_lshr_fshr(i32 %x, i32 %y, i32 %s) {
;
; CHECK-GI-LABEL: or_lshr_fshr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: lsl w8, w1, #1
-; CHECK-GI-NEXT: lsr w9, w1, w2
-; CHECK-GI-NEXT: mvn w10, w2
-; CHECK-GI-NEXT: lsr w11, w0, w2
-; CHECK-GI-NEXT: lsl w8, w8, w10
-; CHECK-GI-NEXT: orr w9, w11, w9
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: and w9, w2, #0x1f
+; CHECK-GI-NEXT: lsl w10, w1, #1
+; CHECK-GI-NEXT: lsr w11, w1, w2
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsr w9, w0, w9
+; CHECK-GI-NEXT: lsl w8, w10, w8
+; CHECK-GI-NEXT: orr w9, w9, w11
; CHECK-GI-NEXT: orr w0, w8, w9
; CHECK-GI-NEXT: ret
%shy = lshr i32 %y, %s
@@ -650,12 +674,14 @@ define i32 @or_lshr_fshr_commute(i32 %x, i32 %y, i32 %s) {
;
; CHECK-GI-LABEL: or_lshr_fshr_commute:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: lsl w8, w1, #1
-; CHECK-GI-NEXT: lsr w9, w1, w2
-; CHECK-GI-NEXT: mvn w10, w2
-; CHECK-GI-NEXT: lsr w11, w0, w2
-; CHECK-GI-NEXT: lsl w8, w8, w10
-; CHECK-GI-NEXT: orr w9, w9, w11
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: and w9, w2, #0x1f
+; CHECK-GI-NEXT: lsl w10, w1, #1
+; CHECK-GI-NEXT: lsr w11, w1, w2
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsr w9, w0, w9
+; CHECK-GI-NEXT: lsl w8, w10, w8
+; CHECK-GI-NEXT: orr w9, w11, w9
; CHECK-GI-NEXT: orr w0, w9, w8
; CHECK-GI-NEXT: ret
%shy = lshr i32 %y, %s
@@ -678,14 +704,25 @@ define i32 @or_lshr_rotr_commute(i32 %x, i32 %y, i32 %s) {
}
define i32 @or_shl_fshl_simplify(i32 %x, i32 %y, i32 %s) {
-; CHECK-LABEL: or_shl_fshl_simplify:
-; CHECK: // %bb.0:
-; CHECK-NEXT: lsr w8, w0, #1
-; CHECK-NEXT: mvn w9, w2
-; CHECK-NEXT: lsl w10, w1, w2
-; CHECK-NEXT: lsr w8, w8, w9
-; CHECK-NEXT: orr w0, w10, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: or_shl_fshl_simplify:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: lsr w8, w0, #1
+; CHECK-SD-NEXT: mvn w9, w2
+; CHECK-SD-NEXT: lsl w10, w1, w2
+; CHECK-SD-NEXT: lsr w8, w8, w9
+; CHECK-SD-NEXT: orr w0, w10, w8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: or_shl_fshl_simplify:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: lsr w9, w0, #1
+; CHECK-GI-NEXT: and w10, w2, #0x1f
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsl w10, w1, w10
+; CHECK-GI-NEXT: lsr w8, w9, w8
+; CHECK-GI-NEXT: orr w0, w10, w8
+; CHECK-GI-NEXT: ret
%shy = shl i32 %y, %s
%fun = call i32 @llvm.fshl.i32(i32 %y, i32 %x, i32 %s)
%or = or i32 %fun, %shy
@@ -704,10 +741,12 @@ define i32 @or_lshr_fshr_simplify(i32 %x, i32 %y, i32 %s) {
;
; CHECK-GI-LABEL: or_lshr_fshr_simplify:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: lsl w8, w0, #1
-; CHECK-GI-NEXT: mvn w9, w2
-; CHECK-GI-NEXT: lsl w8, w8, w9
-; CHECK-GI-NEXT: lsr w9, w1, w2
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: lsl w9, w0, #1
+; CHECK-GI-NEXT: and w10, w2, #0x1f
+; CHECK-GI-NEXT: bic w8, w8, w2
+; CHECK-GI-NEXT: lsl w8, w9, w8
+; CHECK-GI-NEXT: lsr w9, w1, w10
; CHECK-GI-NEXT: orr w0, w8, w9
; CHECK-GI-NEXT: ret
%shy = lshr i32 %y, %s
diff --git a/llvm/test/CodeGen/AArch64/shift-mod.ll b/llvm/test/CodeGen/AArch64/shift-mod.ll
index e53ba2554ac77..7679240666c7b 100644
--- a/llvm/test/CodeGen/AArch64/shift-mod.ll
+++ b/llvm/test/CodeGen/AArch64/shift-mod.ll
@@ -46,11 +46,17 @@ define i64 @test3(i64 %x, i64 %y) {
}
define i64 @test4(i64 %y, i32 %s) {
-; CHECK-LABEL: test4:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT: asr x0, x0, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test4:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT: asr x0, x0, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test4:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov w8, w1
+; CHECK-GI-NEXT: asr x0, x0, x8
+; CHECK-GI-NEXT: ret
entry:
%sh_prom = zext i32 %s to i64
%shr = ashr i64 %y, %sh_prom
@@ -58,11 +64,18 @@ entry:
}
define i64 @test5(i64 %y, i32 %s) {
-; CHECK-LABEL: test5:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT: asr x0, x0, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test5:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT: asr x0, x0, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test5:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT: sxtw x8, w1
+; CHECK-GI-NEXT: asr x0, x0, x8
+; CHECK-GI-NEXT: ret
entry:
%sh_prom = sext i32 %s to i64
%shr = ashr i64 %y, %sh_prom
@@ -70,11 +83,18 @@ entry:
}
define i64 @test6(i64 %y, i32 %s) {
-; CHECK-LABEL: test6:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT: lsl x0, x0, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test6:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT: lsl x0, x0, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test6:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT: sxtw x8, w1
+; CHECK-GI-NEXT: lsl x0, x0, x8
+; CHECK-GI-NEXT: ret
entry:
%sh_prom = sext i32 %s to i64
%shr = shl i64 %y, %sh_prom
@@ -290,3 +310,26 @@ define i64 @shl_i64_add64(i64 %x, i64 %amt) {
%r = shl i64 %x, %add
ret i64 %r
}
+
+; Test SUB N-X where N == 0 mod size generates NEG.
+define i32 @shl_i32_sub64(i32 %x, i32 %amt) {
+; CHECK-LABEL: shl_i32_sub64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: neg w8, w1
+; CHECK-NEXT: lsl w0, w0, w8
+; CHECK-NEXT: ret
+ %sub = sub i32 64, %amt
+ %r = shl i32 %x, %sub
+ ret i32 %r
+}
+
+define i64 @lshr_i64_sub64(i64 %x, i64 %amt) {
+; CHECK-LABEL: lshr_i64_sub64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: neg x8, x1
+; CHECK-NEXT: lsr x0, x0, x8
+; CHECK-NEXT: ret
+ %sub = sub i64 64, %amt
+ %r = lshr i64 %x, %sub
+ ret i64 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/shift.ll b/llvm/test/CodeGen/AArch64/shift.ll
index 2fcdf4df41931..445e6563165d6 100644
--- a/llvm/test/CodeGen/AArch64/shift.ll
+++ b/llvm/test/CodeGen/AArch64/shift.ll
@@ -70,12 +70,11 @@ define i128 @shl_i128(i128 %0, i128 %1){
;
; CHECK-GI-LABEL: shl_i128:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
+; CHECK-GI-NEXT: neg x8, x2
; CHECK-GI-NEXT: lsl x9, x1, x2
; CHECK-GI-NEXT: lsl x10, x0, x2
-; CHECK-GI-NEXT: sub x8, x8, x2
-; CHECK-GI-NEXT: cmp x2, #64
; CHECK-GI-NEXT: lsr x8, x0, x8
+; CHECK-GI-NEXT: cmp x2, #64
; CHECK-GI-NEXT: csel x0, x10, xzr, lo
; CHECK-GI-NEXT: orr x8, x8, x9
; CHECK-GI-NEXT: csel x8, x8, x10, lo
@@ -158,12 +157,11 @@ define i128 @ashr_i128(i128 %0, i128 %1){
;
; CHECK-GI-LABEL: ashr_i128:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
+; CHECK-GI-NEXT: neg x8, x2
; CHECK-GI-NEXT: lsr x9, x0, x2
; CHECK-GI-NEXT: asr x10, x1, x2
-; CHECK-GI-NEXT: sub x8, x8, x2
-; CHECK-GI-NEXT: cmp x2, #64
; CHECK-GI-NEXT: lsl x8, x1, x8
+; CHECK-GI-NEXT: cmp x2, #64
; CHECK-GI-NEXT: orr x8, x9, x8
; CHECK-GI-NEXT: asr x9, x1, #63
; CHECK-GI-NEXT: csel x8, x8, x10, lo
@@ -246,12 +244,11 @@ define i128 @lshr_i128(i128 %0, i128 %1){
;
; CHECK-GI-LABEL: lshr_i128:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
+; CHECK-GI-NEXT: neg x8, x2
; CHECK-GI-NEXT: lsr x9, x0, x2
; CHECK-GI-NEXT: lsr x10, x1, x2
-; CHECK-GI-NEXT: sub x8, x8, x2
-; CHECK-GI-NEXT: cmp x2, #64
; CHECK-GI-NEXT: lsl x8, x1, x8
+; CHECK-GI-NEXT: cmp x2, #64
; CHECK-GI-NEXT: orr x8, x9, x8
; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x2, #0
@@ -613,25 +610,24 @@ define <2 x i128> @shl_v2i128(<2 x i128> %0, <2 x i128> %1){
;
; CHECK-GI-LABEL: shl_v2i128:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: lsl x10, x1, x4
-; CHECK-GI-NEXT: lsl x11, x0, x4
-; CHECK-GI-NEXT: sub x9, x8, x4
-; CHECK-GI-NEXT: sub x8, x8, x6
+; CHECK-GI-NEXT: neg x8, x4
+; CHECK-GI-NEXT: lsl x9, x1, x4
+; CHECK-GI-NEXT: lsl x10, x0, x4
+; CHECK-GI-NEXT: lsr x8, x0, x8
+; CHECK-GI-NEXT: neg x11, x6
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: lsr x9, x0, x9
-; CHECK-GI-NEXT: lsr x8, x2, x8
-; CHECK-GI-NEXT: csel x0, x11, xzr, lo
-; CHECK-GI-NEXT: orr x9, x9, x10
-; CHECK-GI-NEXT: lsl x10, x3, x6
-; CHECK-GI-NEXT: csel x9, x9, x11, lo
+; CHECK-GI-NEXT: lsr x11, x2, x11
+; CHECK-GI-NEXT: csel x0, x10, xzr, lo
+; CHECK-GI-NEXT: orr x8, x8, x9
+; CHECK-GI-NEXT: lsl x9, x3, x6
+; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x4, #0
-; CHECK-GI-NEXT: lsl x11, x2, x6
-; CHECK-GI-NEXT: csel x1, x1, x9, eq
-; CHECK-GI-NEXT: orr x8, x8, x10
+; CHECK-GI-NEXT: lsl x10, x2, x6
+; CHECK-GI-NEXT: csel x1, x1, x8, eq
+; CHECK-GI-NEXT: orr x8, x11, x9
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: csel x2, x11, xzr, lo
-; CHECK-GI-NEXT: csel x8, x8, x11, lo
+; CHECK-GI-NEXT: csel x2, x10, xzr, lo
+; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x6, #0
; CHECK-GI-NEXT: csel x3, x3, x8, eq
; CHECK-GI-NEXT: ret
@@ -777,31 +773,30 @@ define <2 x i128> @ashr_v2i128(<2 x i128> %0, <2 x i128> %1){
;
; CHECK-GI-LABEL: ashr_v2i128:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: lsr x10, x0, x4
-; CHECK-GI-NEXT: asr x11, x1, x4
-; CHECK-GI-NEXT: sub x9, x8, x4
+; CHECK-GI-NEXT: neg x8, x4
+; CHECK-GI-NEXT: lsr x9, x0, x4
+; CHECK-GI-NEXT: asr x10, x1, x4
+; CHECK-GI-NEXT: lsl x8, x1, x8
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: sub x8, x8, x6
-; CHECK-GI-NEXT: lsl x9, x1, x9
+; CHECK-GI-NEXT: neg x11, x6
; CHECK-GI-NEXT: lsr x12, x2, x6
-; CHECK-GI-NEXT: lsl x8, x3, x8
-; CHECK-GI-NEXT: orr x9, x10, x9
-; CHECK-GI-NEXT: asr x10, x1, #63
-; CHECK-GI-NEXT: orr x8, x12, x8
-; CHECK-GI-NEXT: csel x9, x9, x11, lo
+; CHECK-GI-NEXT: lsl x11, x3, x11
+; CHECK-GI-NEXT: orr x8, x9, x8
+; CHECK-GI-NEXT: asr x9, x1, #63
+; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x4, #0
-; CHECK-GI-NEXT: csel x0, x0, x9, eq
+; CHECK-GI-NEXT: csel x0, x0, x8, eq
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: asr x9, x3, x6
-; CHECK-GI-NEXT: csel x1, x11, x10, lo
+; CHECK-GI-NEXT: asr x8, x3, x6
+; CHECK-GI-NEXT: csel x1, x10, x9, lo
+; CHECK-GI-NEXT: orr x9, x12, x11
; CHECK-GI-NEXT: cmp x6, #64
; CHECK-GI-NEXT: asr x10, x3, #63
-; CHECK-GI-NEXT: csel x8, x8, x9, lo
+; CHECK-GI-NEXT: csel x9, x9, x8, lo
; CHECK-GI-NEXT: cmp x6, #0
-; CHECK-GI-NEXT: csel x2, x2, x8, eq
+; CHECK-GI-NEXT: csel x2, x2, x9, eq
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: csel x3, x9, x10, lo
+; CHECK-GI-NEXT: csel x3, x8, x10, lo
; CHECK-GI-NEXT: ret
%3 = ashr <2 x i128> %0, %1
ret <2 x i128> %3
@@ -941,29 +936,28 @@ define <2 x i128> @lshr_v2i128(<2 x i128> %0, <2 x i128> %1){
;
; CHECK-GI-LABEL: lshr_v2i128:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov w8, #64 // =0x40
-; CHECK-GI-NEXT: lsr x10, x0, x4
-; CHECK-GI-NEXT: lsr x11, x1, x4
-; CHECK-GI-NEXT: sub x9, x8, x4
+; CHECK-GI-NEXT: neg x8, x4
+; CHECK-GI-NEXT: lsr x9, x0, x4
+; CHECK-GI-NEXT: lsr x10, x1, x4
+; CHECK-GI-NEXT: lsl x8, x1, x8
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: sub x8, x8, x6
-; CHECK-GI-NEXT: lsl x9, x1, x9
-; CHECK-GI-NEXT: lsl x8, x3, x8
-; CHECK-GI-NEXT: orr x9, x10, x9
-; CHECK-GI-NEXT: lsr x10, x2, x6
-; CHECK-GI-NEXT: csel x9, x9, x11, lo
+; CHECK-GI-NEXT: lsr x11, x2, x6
+; CHECK-GI-NEXT: orr x8, x9, x8
+; CHECK-GI-NEXT: neg x9, x6
+; CHECK-GI-NEXT: csel x8, x8, x10, lo
; CHECK-GI-NEXT: cmp x4, #0
-; CHECK-GI-NEXT: csel x0, x0, x9, eq
+; CHECK-GI-NEXT: lsl x9, x3, x9
+; CHECK-GI-NEXT: csel x0, x0, x8, eq
; CHECK-GI-NEXT: cmp x4, #64
-; CHECK-GI-NEXT: lsr x9, x3, x6
-; CHECK-GI-NEXT: csel x1, x11, xzr, lo
-; CHECK-GI-NEXT: orr x8, x10, x8
+; CHECK-GI-NEXT: lsr x8, x3, x6
+; CHECK-GI-NEXT: csel x1, x10, xzr, lo
+; CHECK-GI-NEXT: orr x9, x11, x9
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: csel x8, x8, x9, lo
+; CHECK-GI-NEXT: csel x9, x9, x8, lo
; CHECK-GI-NEXT: cmp x6, #0
-; CHECK-GI-NEXT: csel x2, x2, x8, eq
+; CHECK-GI-NEXT: csel x2, x2, x9, eq
; CHECK-GI-NEXT: cmp x6, #64
-; CHECK-GI-NEXT: csel x3, x9, xzr, lo
+; CHECK-GI-NEXT: csel x3, x8, xzr, lo
; CHECK-GI-NEXT: ret
%3 = lshr <2 x i128> %0, %1
ret <2 x i128> %3
>From c3358b589a4f9c77170110a4f81cfd12436cf379 Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Fri, 25 Sep 2026 00:35:18 +0530
Subject: [PATCH 2/4] AArch64: Generate MVN for shift by (N - X) in
SelectShiftMask
Extend SelectShiftMask to handle SUB where the constant operand is
-1 mod the shift width: shl x, (sub N, y) where N % ShiftWidth ==
ShiftWidth-1 becomes shl x, not(y), generating a NOT (MVN) instead
of a SUB from a constant.
For example, shl x, (sub 63, y) becomes shl x, mvn(y).
This implements both SelectionDAG (ORNWrr/ORNXrr with the zero
register) and GlobalISel (built in the ComplexPattern renderer and
constrained with constrainSelectedInstRegOperands).
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 20 ++++++++++++++++
.../GISel/AArch64InstructionSelector.cpp | 24 +++++++++++++++++++
llvm/test/CodeGen/AArch64/shift-mod.ll | 23 ++++++++++++++++++
3 files changed, 67 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 31bedd86859a9..9628001dc712d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -822,6 +822,26 @@ bool AArch64DAGToDAGISel::SelectShiftMask(SDValue N, SDValue &ShAmt) {
}
}
+ // If shifting by N-X where N == -1 mod ShiftWidth, then just shift by ~X
+ // to generate a NOT (MVN) instead of a SUB from a constant.
+ if (N.getOpcode() == ISD::SUB && N.hasOneUse() &&
+ N.getValueType() == (ShiftWidth == 32 ? MVT::i32 : MVT::i64)) {
+ uint64_t Imm;
+ if (isIntImmediate(N.getOperand(0).getNode(), Imm) &&
+ (Imm % ShiftWidth == ShiftWidth - 1)) {
+ SDLoc DL(N);
+ EVT VT = N.getValueType();
+ unsigned NotOpc = (ShiftWidth == 32) ? AArch64::ORNWrr : AArch64::ORNXrr;
+ unsigned ZeroReg = (ShiftWidth == 32) ? AArch64::WZR : AArch64::XZR;
+ SDValue Zero =
+ CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, VT);
+ MachineSDNode *Not =
+ CurDAG->getMachineNode(NotOpc, DL, VT, Zero, N.getOperand(1));
+ ShAmt = SDValue(Not, 0);
+ return true;
+ }
+ }
+
return false;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 29c1f32911d3c..4a8638c89d03b 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -7377,6 +7377,30 @@ AArch64InstructionSelector::selectShiftMask(MachineOperand &Root) const {
}}};
}
+ // If shifting by N-X where N == -1 mod ShiftWidth, then just shift by ~X
+ // to generate a NOT (MVN) instead of a SUB from a constant.
+ Register NotSrcReg;
+ int64_t NotImm;
+ if (MRI.hasOneUse(ShAmtReg) &&
+ mi_match(ShAmtReg, MRI, m_GSub(m_ICst(NotImm), m_Reg(NotSrcReg))) &&
+ (NotImm % ShiftWidth == ShiftWidth - 1)) {
+ return {{[=](MachineInstrBuilder &MIB) {
+ MachineInstr *I = MIB.getInstr();
+ MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
+ const TargetRegisterClass &RC =
+ ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
+ unsigned NotOpc = ShiftWidth == 32 ? AArch64::ORNWrr : AArch64::ORNXrr;
+ Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
+ Register NotReg = MRI2.createVirtualRegister(&RC);
+ auto NotMI = BuildMI(*I->getParent(), *I, I->getDebugLoc(),
+ TII.get(NotOpc), NotReg)
+ .addReg(ZeroReg)
+ .addReg(NotSrcReg);
+ constrainSelectedInstRegOperands(*NotMI, TII, TRI, RBI);
+ MIB.addReg(NotReg);
+ }}};
+ }
+
// Only succeed if we changed the shift amount; otherwise let other
// patterns (e.g. zext GPR32 -> SUBREG_TO_REG) match instead.
if (ShAmtReg == Root.getReg())
diff --git a/llvm/test/CodeGen/AArch64/shift-mod.ll b/llvm/test/CodeGen/AArch64/shift-mod.ll
index 7679240666c7b..9e81f4ad3ce74 100644
--- a/llvm/test/CodeGen/AArch64/shift-mod.ll
+++ b/llvm/test/CodeGen/AArch64/shift-mod.ll
@@ -333,3 +333,26 @@ define i64 @lshr_i64_sub64(i64 %x, i64 %amt) {
%r = lshr i64 %x, %sub
ret i64 %r
}
+
+; Test SUB N-X where N == -1 mod size generates MVN.
+define i32 @shl_i32_sub63(i32 %x, i32 %amt) {
+; CHECK-LABEL: shl_i32_sub63:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvn w8, w1
+; CHECK-NEXT: lsl w0, w0, w8
+; CHECK-NEXT: ret
+ %sub = sub i32 63, %amt
+ %r = shl i32 %x, %sub
+ ret i32 %r
+}
+
+define i64 @lshr_i64_sub63(i64 %x, i64 %amt) {
+; CHECK-LABEL: lshr_i64_sub63:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvn x8, x1
+; CHECK-NEXT: lsr x0, x0, x8
+; CHECK-NEXT: ret
+ %sub = sub i64 63, %amt
+ %r = lshr i64 %x, %sub
+ ret i64 %r
+}
>From da3164f13150f64f19b5594079adc407147ae244 Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Sat, 26 Sep 2026 01:21:05 +0530
Subject: [PATCH 3/4] AArch64: Support ROTR shift amount masking in
SelectShiftMask
The RORV instructions already use the Shift multiclass, so they inherit
the shiftMask32/shiftMask64 ComplexPattern patterns. Add an additional
pattern for the 32-bit rotate case where the shift amount is legalized
with a zext to i64, peeking through the zext to apply the 32-bit
shift-amount masking.
---
.../lib/Target/AArch64/AArch64InstrFormats.td | 5 ++++
llvm/test/CodeGen/AArch64/shift-mod.ll | 30 +++++++++++++++++++
2 files changed, 35 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index fc603c2d59240..0277ba7cee9dd 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -2788,6 +2788,11 @@ multiclass Shift<bits<2> shift_type, string asm, SDNode OpNode> {
(!cast<Instruction>(NAME # "Wr") GPR32:$Rn, shiftMask32:$Rm)>;
def : Pat<(i64 (OpNode GPR64:$Rn, shiftMask64:$Rm)),
(!cast<Instruction>(NAME # "Xr") GPR64:$Rn, shiftMask64:$Rm)>;
+
+ // For 32-bit rotates the shift amount is legalized with a zext to i64;
+ // peek through it and apply the 32-bit shift-amount masking.
+ def : Pat<(i32 (OpNode GPR32:$Rn, (i64 (zext (i32 shiftMask32:$Rm))))),
+ (!cast<Instruction>(NAME # "Wr") GPR32:$Rn, shiftMask32:$Rm)>;
}
class ShiftAlias<string asm, Instruction inst, RegisterClass regtype>
diff --git a/llvm/test/CodeGen/AArch64/shift-mod.ll b/llvm/test/CodeGen/AArch64/shift-mod.ll
index 9e81f4ad3ce74..f5d42c152d61e 100644
--- a/llvm/test/CodeGen/AArch64/shift-mod.ll
+++ b/llvm/test/CodeGen/AArch64/shift-mod.ll
@@ -356,3 +356,33 @@ define i64 @lshr_i64_sub63(i64 %x, i64 %amt) {
%r = lshr i64 %x, %sub
ret i64 %r
}
+
+; Test rotate with masked/modified shift amount.
+define i32 @rotr_i32_and(i32 %x, i32 %amt) {
+; CHECK-SD-LABEL: rotr_i32_and:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: ror w0, w0, w1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: rotr_i32_and:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: and w8, w1, #0x1f
+; CHECK-GI-NEXT: ror w0, w0, w8
+; CHECK-GI-NEXT: ret
+ %m = and i32 %amt, 31
+ %r = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %m)
+ ret i32 %r
+}
+
+define i32 @rotr_i32_sub(i32 %x, i32 %amt) {
+; CHECK-LABEL: rotr_i32_sub:
+; CHECK: // %bb.0:
+; CHECK-NEXT: neg w8, w1
+; CHECK-NEXT: ror w0, w0, w8
+; CHECK-NEXT: ret
+ %s = sub i32 64, %amt
+ %r = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %s)
+ ret i32 %r
+}
+
+declare i32 @llvm.fshr.i32(i32, i32, i32)
>From 929077a7b229461f8b549318a71b1b559c86ec96 Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Sat, 3 Oct 2026 23:27:48 +0530
Subject: [PATCH 4/4] AArch64: Remove tryShiftAmountMod (WIP - find remaining
gaps)
Removes tryShiftAmountMod now that SelectShiftMask ComplexPattern handles
the shift amount optimizations. This is a draft to surface the remaining
cases the ComplexPattern does not yet cover, tracked in #224245.
Known remaining gaps:
- i8/i16 AND masks: (i32 (shl GPR32, (i64 (and X, 0xff))))
- Multi-use NEG: sub N, x feeding multiple shifts
- ROTR via zext: sub N, x for 32-bit rotate zext'd to i64
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 128 ------------------
1 file changed, 128 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 9628001dc712d..f2b80134dee2a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -448,7 +448,6 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool tryBitfieldExtractOpFromSExt(SDNode *N);
bool tryBitfieldInsertOp(SDNode *N);
bool tryBitfieldInsertInZeroOp(SDNode *N);
- bool tryShiftAmountMod(SDNode *N);
bool tryReadRegister(SDNode *N);
bool tryWriteRegister(SDNode *N);
@@ -4220,131 +4219,6 @@ bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
return true;
}
-/// tryShiftAmountMod - Take advantage of built-in mod of shift amount in
-/// variable shift/rotate instructions.
-bool AArch64DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
- EVT VT = N->getValueType(0);
-
- unsigned Opc;
- switch (N->getOpcode()) {
- case ISD::ROTR:
- Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
- break;
- case ISD::SHL:
- Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
- break;
- case ISD::SRL:
- Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
- break;
- case ISD::SRA:
- Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
- break;
- default:
- return false;
- }
-
- uint64_t Size;
- uint64_t Bits;
- if (VT == MVT::i32) {
- Bits = 5;
- Size = 32;
- } else if (VT == MVT::i64) {
- Bits = 6;
- Size = 64;
- } else
- return false;
-
- SDValue ShiftAmt = N->getOperand(1);
- SDLoc DL(N);
- SDValue NewShiftAmt;
-
- // Skip over an extend of the shift amount.
- if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND ||
- ShiftAmt->getOpcode() == ISD::ANY_EXTEND)
- ShiftAmt = ShiftAmt->getOperand(0);
-
- if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) {
- SDValue Add0 = ShiftAmt->getOperand(0);
- SDValue Add1 = ShiftAmt->getOperand(1);
- uint64_t Add0Imm;
- uint64_t Add1Imm;
- if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0)) {
- // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X
- // to avoid the ADD/SUB.
- NewShiftAmt = Add0;
- } else if (ShiftAmt->getOpcode() == ISD::SUB &&
- isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 &&
- (Add0Imm % Size == 0)) {
- // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
- // to generate a NEG instead of a SUB from a constant.
- unsigned NegOpc;
- unsigned ZeroReg;
- EVT SubVT = ShiftAmt->getValueType(0);
- if (SubVT == MVT::i32) {
- NegOpc = AArch64::SUBWrr;
- ZeroReg = AArch64::WZR;
- } else {
- assert(SubVT == MVT::i64);
- NegOpc = AArch64::SUBXrr;
- ZeroReg = AArch64::XZR;
- }
- SDValue Zero =
- CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT);
- MachineSDNode *Neg =
- CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1);
- NewShiftAmt = SDValue(Neg, 0);
- } else if (ShiftAmt->getOpcode() == ISD::SUB &&
- isIntImmediate(Add0, Add0Imm) && (Add0Imm % Size == Size - 1)) {
- // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
- // to generate a NOT instead of a SUB from a constant.
- unsigned NotOpc;
- unsigned ZeroReg;
- EVT SubVT = ShiftAmt->getValueType(0);
- if (SubVT == MVT::i32) {
- NotOpc = AArch64::ORNWrr;
- ZeroReg = AArch64::WZR;
- } else {
- assert(SubVT == MVT::i64);
- NotOpc = AArch64::ORNXrr;
- ZeroReg = AArch64::XZR;
- }
- SDValue Zero =
- CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT);
- MachineSDNode *Not =
- CurDAG->getMachineNode(NotOpc, DL, SubVT, Zero, Add1);
- NewShiftAmt = SDValue(Not, 0);
- } else
- return false;
- } else {
- // If the shift amount is masked with an AND, check that the mask covers the
- // bits that are implicitly ANDed off by the above opcodes and if so, skip
- // the AND.
- uint64_t MaskImm;
- if (!isOpcWithIntImmediate(ShiftAmt.getNode(), ISD::AND, MaskImm) &&
- !isOpcWithIntImmediate(ShiftAmt.getNode(), AArch64ISD::ANDS, MaskImm))
- return false;
-
- if ((unsigned)llvm::countr_one(MaskImm) < Bits)
- return false;
-
- NewShiftAmt = ShiftAmt->getOperand(0);
- }
-
- // Narrow/widen the shift amount to match the size of the shift operation.
- if (VT == MVT::i32)
- NewShiftAmt = narrowIfNeeded(CurDAG, NewShiftAmt);
- else if (VT == MVT::i64 && NewShiftAmt->getValueType(0) == MVT::i32) {
- SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, DL, MVT::i32);
- MachineSDNode *Ext = CurDAG->getMachineNode(AArch64::SUBREG_TO_REG, DL, VT,
- NewShiftAmt, SubReg);
- NewShiftAmt = SDValue(Ext, 0);
- }
-
- SDValue Ops[] = {N->getOperand(0), NewShiftAmt};
- CurDAG->SelectNodeTo(N, Opc, VT, Ops);
- return true;
-}
-
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N,
SDValue &FixedPos,
unsigned RegWidth,
@@ -5322,8 +5196,6 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
[[fallthrough]];
case ISD::ROTR:
case ISD::SHL:
- if (tryShiftAmountMod(Node))
- return;
break;
case ISD::SIGN_EXTEND:
More information about the llvm-commits
mailing list