[llvm] [AArch64][GlobalISel] Add pre-legalizer combines for AVGFLOOR and AVGCEIL (PR #192866)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 19 12:09:05 PDT 2026
https://github.com/Atafid created https://github.com/llvm/llvm-project/pull/192866
This patch adds GlobalISel pre-legalizer combines to pattern-match and optimize average operations, bringing GlobalISel on par with SelectionDAG.
Specifically, it matches:
- `(a + b) >> 1` into `G_UAVGFLOOR` / `G_SAVGFLOOR`
- `(a + b + 1) >> 1` into `G_UAVGCEIL` / `G_SAVGCEIL`
Support is included for both scalar and vector types, correctly handling constants and splat vectors via `isOneOrOneSplat()`. This builds upon the generic opcodes introduced for AArch64 intrinsics lowering and enables optimal emission of Neon instructions (e.g., `urhadd`, `shadd`) directly from generic IR.
Fixes #118083
>From 36a55b5827a6f7fbd0805fad6d3f4aa3343183a6 Mon Sep 17 00:00:00 2001
From: Guillaume Di Fatta <difatta.guillaume at gmail.com>
Date: Sun, 19 Apr 2026 21:04:20 +0200
Subject: [PATCH] [GlobalISel] Add pre-legalizer combines for AVGFLOOR and
AVGCEIL
This patch adds GlobalISel pre-legalizer combines to pattern-match and
optimize average operations, bringing GlobalISel on par with SelectionDAG.
Specifically, it matches:
- `(a + b) >> 1` into `G_UAVGFLOOR` / `G_SAVGFLOOR`
- `(a + b + 1) >> 1` into `G_UAVGCEIL` / `G_SAVGCEIL`
Support is included for both scalar and vector types, correctly handling
constants and splat vectors via `isOneOrOneSplat()`. This builds upon
the generic opcodes introduced for AArch64 intrinsics lowering and
enables optimal emission of Neon instructions (e.g., `urhadd`, `shadd`)
directly from generic IR.
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 8 +
.../include/llvm/Target/GlobalISel/Combine.td | 44 +-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 145 +++++
.../AArch64/GlobalISel/combine-avg.mir | 319 ++++++++++
llvm/test/CodeGen/AArch64/arm64-vhadd.ll | 594 +++++-------------
5 files changed, 655 insertions(+), 455 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 365bbaacfe055..294de3747b9bc 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -1054,6 +1054,14 @@ class CombinerHelper {
// (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
bool matchCtls(MachineInstr &CtlzMI, BuildFnTy &MatchInfo) const;
+ // shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
+ bool matchAVGFloor(MachineInstr &MI, MachineRegisterInfo &MRI,
+ BuildFnTy &MatchInfo) const;
+
+ // shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
+ bool matchAVGCeil(MachineInstr &MI, MachineRegisterInfo &MRI,
+ BuildFnTy &MatchInfo) const;
+
private:
/// Checks for legality of an indexed variant of \p LdSt.
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index df3955909fe85..ea90410eb03c3 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1174,29 +1174,29 @@ def funnel_shift_overshift: GICombineRule<
// Transform: fshl x, z, y | shl x, y -> fshl x, z, y
// Transform: shl x, y | fshl x, z, y -> fshl x, z, y
-// FIXME: TableGen didn't handle G_OR commutativity on its own,
+// FIXME: TableGen didn't handle G_OR commutativity on its own,
// necessitating the use of !foreach to handle it manually.
def funnel_shift_or_shift_to_funnel_shift_left_frags : GICombinePatFrag<
(outs root: $dst, $out1, $out2), (ins),
- !foreach(inst, [(G_OR $dst, $out1, $out2), (G_OR $dst, $out2, $out1)],
+ !foreach(inst, [(G_OR $dst, $out1, $out2), (G_OR $dst, $out2, $out1)],
(pattern (G_FSHL $out1, $x, $z, $y), (G_SHL $out2, $x, $y), inst))>;
def funnel_shift_or_shift_to_funnel_shift_left: GICombineRule<
- (defs root:$root),
+ (defs root:$root),
(match (funnel_shift_or_shift_to_funnel_shift_left_frags $root, $out1, $out2)),
(apply (GIReplaceReg $root, $out1))
>;
// Transform: fshr z, x, y | srl x, y -> fshr z, x, y
// Transform: srl x, y | fshr z, x, y -> fshr z, x, y
-// FIXME: TableGen didn't handle G_OR commutativity on its own,
+// FIXME: TableGen didn't handle G_OR commutativity on its own,
// necessitating the use of !foreach to handle it manually.
def funnel_shift_or_shift_to_funnel_shift_right_frags : GICombinePatFrag<
(outs root: $dst, $out1, $out2), (ins),
- !foreach(inst, [(G_OR $dst, $out1, $out2), (G_OR $dst, $out2, $out1)],
+ !foreach(inst, [(G_OR $dst, $out1, $out2), (G_OR $dst, $out2, $out1)],
(pattern (G_FSHR $out1, $z, $x, $y), (G_LSHR $out2, $x, $y), inst))>;
def funnel_shift_or_shift_to_funnel_shift_right: GICombineRule<
- (defs root:$root),
- (match (funnel_shift_or_shift_to_funnel_shift_right_frags $root, $out1, $out2)),
+ (defs root:$root),
+ (match (funnel_shift_or_shift_to_funnel_shift_right_frags $root, $out1, $out2)),
(apply (GIReplaceReg $root, $out1))
>;
@@ -1323,7 +1323,7 @@ def udiv_by_pow2 : GICombineRule<
[{ return Helper.matchDivByPow2(*${root}, /*IsSigned=*/false); }]),
(apply [{ Helper.applyUDivByPow2(*${root}); }])>;
-def intdiv_combines : GICombineGroup<[udiv_by_pow2, sdiv_by_pow2,
+def intdiv_combines : GICombineGroup<[udiv_by_pow2, sdiv_by_pow2,
udiv_by_const, sdiv_by_const,]>;
def urem_by_const : GICombineRule<
@@ -1434,7 +1434,7 @@ def trunc_ssatu : GICombineRule<
def trunc_usatu : GICombineRule<
(defs root:$root),
- (match (G_UMIN $min, $x, $y):$Min,
+ (match (G_UMIN $min, $x, $y):$Min,
(G_TRUNC $dst, $min):$root,
[{ return Helper.matchTruncUSatU(*${root}, *${Min}); }]),
(apply (G_TRUNC_USAT_U $dst, $x))>;
@@ -1982,14 +1982,14 @@ def APlusBMinusAplusC : GICombineRule<
(G_SUB $sub1, $B, $add1),
(G_ADD $root, $A, $sub1)),
(apply (G_SUB $root, $B, $C))>;
-
+
// fold (A-(B-C)) to A+(C-B)
def AMinusBMinusC : GICombineRule<
(defs root:$root),
(match (G_SUB $sub1, $B, $C),
- (G_SUB $root, $A, $sub1),
+ (G_SUB $root, $A, $sub1),
[{ return MRI.hasOneNonDBGUse(${sub1}.getReg()); }]),
- (apply (G_SUB $add, $C, $B),
+ (apply (G_SUB $add, $C, $B),
(G_ADD $root, $A, $add))>;
// fold (A+(B-(C+A))) to (B-C)
@@ -1999,7 +1999,7 @@ def APlusBMinusCPlusA : GICombineRule<
(G_SUB $sub1, $B, $add1),
(G_ADD $root, $A, $sub1)),
(apply (G_SUB $root, $B, $C))>;
-
+
// fold (A - (0 - B)) to (A + B)
def AMinusZeroMinusB : GICombineRule<
(defs root:$root),
@@ -2013,7 +2013,7 @@ def AMinusBMinusA: GICombineRule<
(match (G_SUB $add, $A, $B),
(G_SUB $root, $A, $add)),
(apply (GIReplaceReg $root, $B))>;
-
+
// fold (not (add X, -1)) -> (sub 0, X)
def NotAPlusNegOne: GICombineRule<
(defs root:$root),
@@ -2287,6 +2287,20 @@ def overflow_combines: GICombineGroup<[
match_subo_no_overflow
]>;
+def avgfloor_match : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_LSHR, G_ASHR):$root,
+ [{ return Helper.matchAVGFloor(*${root}, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
+>;
+
+def avgceil_match : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_LSHR, G_ASHR):$root,
+ [{ return Helper.matchAVGCeil(*${root}, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
+>;
+
// FIXME: These should use the custom predicate feature once it lands.
def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
undef_to_negative_one,
@@ -2376,7 +2390,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
combine_use_vector_truncate, merge_combines, overflow_combines,
truncsat_combines, lshr_of_trunc_of_lshr, ctls_combines, add_shift, sub_one_from_sub,
- binop_with_neg, sub_minus_one]>;
+ binop_with_neg, sub_minus_one, avgfloor_match, avgceil_match]>;
// A combine group used to for prelegalizer combiners at -O0. The combines in
// this group have been selected based on experiments to balance code size and
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 0a7d682c44049..6288be25c1c6f 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -8692,3 +8692,148 @@ bool CombinerHelper::matchCtls(MachineInstr &CtlzMI,
return true;
}
+
+static bool matchAnySExt(MachineRegisterInfo &MRI, Register Reg,
+ Register &ExtSrc) {
+ if (mi_match(Reg, MRI, m_GSExt(m_Reg(ExtSrc))))
+ return true;
+
+ MachineInstr *ExtInst = MRI.getVRegDef(Reg);
+ if (ExtInst && ExtInst->getOpcode() == TargetOpcode::G_SEXT_INREG) {
+ ExtSrc = ExtInst->getOperand(1).getReg();
+ return true;
+ }
+ return false;
+};
+
+// Fold shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
+bool CombinerHelper::matchAVGFloor(MachineInstr &MI, MachineRegisterInfo &MRI,
+ BuildFnTy &MatchInfo) const {
+ assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
+ MI.getOpcode() == TargetOpcode::G_ASHR) &&
+ "Expected G_LSHR/G_ASHR");
+
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ Register ShiftAmtReg = MI.getOperand(2).getReg();
+
+ Register X, Y;
+ bool IsSigned = (MI.getOpcode() == TargetOpcode::G_ASHR);
+
+ Register ExtReg1, ExtReg2;
+ if (!mi_match(Src, MRI, m_OneUse(m_GAdd(m_Reg(ExtReg1), m_Reg(ExtReg2)))))
+ return false;
+
+ if (IsSigned) {
+ if (!matchAnySExt(MRI, ExtReg1, X) || !matchAnySExt(MRI, ExtReg2, Y)) {
+ return false;
+ }
+ } else {
+ if (!mi_match(ExtReg1, MRI, m_GZExt(m_Reg(X))) ||
+ !mi_match(ExtReg2, MRI, m_GZExt(m_Reg(Y))))
+ return false;
+ }
+
+ if (!isOneOrOneSplat(ShiftAmtReg, false))
+ return false;
+
+ LLT XTy = MRI.getType(X);
+ LLT YTy = MRI.getType(Y);
+
+ if (XTy != YTy)
+ return false;
+
+ LLT DstTy = MRI.getType(Dst);
+ bool NeedExt = DstTy.getSizeInBits() > XTy.getSizeInBits();
+
+ MatchInfo = [=](MachineIRBuilder &B) {
+ unsigned TargetOpc =
+ IsSigned ? TargetOpcode::G_SAVGFLOOR : TargetOpcode::G_UAVGFLOOR;
+
+ auto Avg = B.buildInstr(TargetOpc, {XTy}, {X, Y});
+
+ if (NeedExt) {
+ if (IsSigned)
+ B.buildSExt(Dst, Avg);
+ else
+ B.buildZExt(Dst, Avg);
+ } else {
+ B.buildCopy(Dst, Avg);
+ }
+ };
+
+ return true;
+}
+
+// Fold shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
+bool CombinerHelper::matchAVGCeil(MachineInstr &MI, MachineRegisterInfo &MRI,
+ BuildFnTy &MatchInfo) const {
+ assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
+ MI.getOpcode() == TargetOpcode::G_ASHR) &&
+ "Expected G_LSHR/G_ASHR");
+
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ Register ShiftAmtReg = MI.getOperand(2).getReg();
+
+ Register X, Y;
+ bool IsSigned = (MI.getOpcode() == TargetOpcode::G_ASHR);
+
+ // check for ((A + B) + 1) or ((A + 1) + B)
+ Register SumLHS, SumRHS;
+ if (!mi_match(Src, MRI, m_GAdd(m_Reg(SumLHS), m_Reg(SumRHS))))
+ return false;
+
+ Register InnerAddReg;
+ if (isOneOrOneSplat(SumLHS, false)) {
+ InnerAddReg = SumRHS;
+ } else if (isOneOrOneSplat(SumRHS, false)) {
+ InnerAddReg = SumLHS;
+ } else {
+ return false;
+ }
+
+ Register ExtReg1, ExtReg2;
+ if (!mi_match(InnerAddReg, MRI, m_GAdd(m_Reg(ExtReg1), m_Reg(ExtReg2))))
+ return false;
+
+ if (IsSigned) {
+ if (!matchAnySExt(MRI, ExtReg1, X) || !matchAnySExt(MRI, ExtReg2, Y)) {
+ return false;
+ }
+ } else {
+ if (!mi_match(ExtReg1, MRI, m_GZExt(m_Reg(X))) ||
+ !mi_match(ExtReg2, MRI, m_GZExt(m_Reg(Y))))
+ return false;
+ }
+
+ if (!isOneOrOneSplat(ShiftAmtReg, false))
+ return false;
+
+ LLT XTy = MRI.getType(X);
+ LLT YTy = MRI.getType(Y);
+
+ if (XTy != YTy)
+ return false;
+
+ LLT DstTy = MRI.getType(Dst);
+ bool NeedExt = DstTy.getSizeInBits() > XTy.getSizeInBits();
+
+ MatchInfo = [=](MachineIRBuilder &B) {
+ unsigned TargetOpc =
+ IsSigned ? TargetOpcode::G_SAVGCEIL : TargetOpcode::G_UAVGCEIL;
+
+ auto Avg = B.buildInstr(TargetOpc, {XTy}, {X, Y});
+
+ if (NeedExt) {
+ if (IsSigned)
+ B.buildSExt(Dst, Avg);
+ else
+ B.buildZExt(Dst, Avg);
+ } else {
+ B.buildCopy(Dst, Avg);
+ }
+ };
+
+ return true;
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
new file mode 100644
index 0000000000000..65bd0901087df
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
@@ -0,0 +1,319 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+...
+---
+name: test_uavgfloor_i16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_uavgfloor_i16
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[UAVGFLOOR:%[0-9]+]]:_(s16) = G_UAVGFLOOR [[TRUNC]], [[TRUNC1]]
+ ; CHECK-NEXT: %ret_ext:_(s32) = G_ANYEXT [[UAVGFLOOR]](s16)
+ ; CHECK-NEXT: $w0 = COPY %ret_ext(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 1
+ %6:_(s32) = G_LSHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgfloor_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_uavgfloor_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: [[UAVGFLOOR:%[0-9]+]]:_(<8 x s16>) = G_UAVGFLOOR [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: $q0 = COPY [[UAVGFLOOR]](<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+
+ %ext0:_(<8 x s32>) = G_ZEXT %0
+ %ext1:_(<8 x s32>) = G_ZEXT %1
+
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_LSHR %sum, %v_shift_cst
+
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_savgfloor_i16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_savgfloor_i16
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[SAVGFLOOR:%[0-9]+]]:_(s32) = G_SAVGFLOOR %w0_copy, %w1_copy
+ ; CHECK-NEXT: $w0 = COPY [[SAVGFLOOR]](s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_SEXT %0(s16)
+ %3:_(s32) = G_SEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 1
+ %6:_(s32) = G_ASHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_savgfloor_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_savgfloor_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: [[SAVGFLOOR:%[0-9]+]]:_(<8 x s16>) = G_SAVGFLOOR [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: $q0 = COPY [[SAVGFLOOR]](<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+
+ %ext0:_(<8 x s32>) = G_SEXT %0
+ %ext1:_(<8 x s32>) = G_SEXT %1
+
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_ASHR %sum, %v_shift_cst
+
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_negative_floor
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_negative_floor
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+ ; CHECK-NEXT: $w0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 2
+ %6:_(s32) = G_LSHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgceil_s32
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_uavgceil_s32
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[UAVGCEIL:%[0-9]+]]:_(s16) = G_UAVGCEIL [[TRUNC]], [[TRUNC1]]
+ ; CHECK-NEXT: %res:_(s32) = G_ZEXT [[UAVGCEIL]](s16)
+ ; CHECK-NEXT: $w0 = COPY %res(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %sum:_(s32) = G_ADD %2, %3
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %add_one:_(s32) = G_ADD %sum, %cst1
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_LSHR %add_one, %cst_shift
+
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_savgceil_s32_associative
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_savgceil_s32_associative
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[SAVGCEIL:%[0-9]+]]:_(s32) = G_SAVGCEIL %w0_copy, %w1_copy
+ ; CHECK-NEXT: $w0 = COPY [[SAVGCEIL]](s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_SEXT %0(s16)
+ %3:_(s32) = G_SEXT %1(s16)
+
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %add_one:_(s32) = G_ADD %2, %cst1
+ %sum:_(s32) = G_ADD %add_one, %3
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_ASHR %sum, %cst_shift
+
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgceil_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_uavgceil_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: [[UAVGCEIL:%[0-9]+]]:_(<8 x s16>) = G_UAVGCEIL [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: $q0 = COPY [[UAVGCEIL]](<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+ %ext0:_(<8 x s32>) = G_ZEXT %0
+ %ext1:_(<8 x s32>) = G_ZEXT %1
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %v_cst1:_(<8 x s32>) = G_BUILD_VECTOR %cst1, %cst1, %cst1, %cst1, %cst1, %cst1, %cst1, %cst1
+ %add_one:_(<8 x s32>) = G_ADD %sum, %v_cst1
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_LSHR %add_one, %v_shift_cst
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_negative_ceil_bad_constant
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_negative_ceil_bad_constant
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: %sum:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: %cst2:_(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: %add_two:_(s32) = G_ADD %sum, %cst2
+ ; CHECK-NEXT: %cst_shift:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %res:_(s32) = G_LSHR %add_two, %cst_shift(s32)
+ ; CHECK-NEXT: $w0 = COPY %res(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+ %sum:_(s32) = G_ADD %2, %3
+ %cst2:_(s32) = G_CONSTANT i32 2
+ %add_two:_(s32) = G_ADD %sum, %cst2
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_LSHR %add_two, %cst_shift
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
index 4723867bc99f0..a1beafe4ef09d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
@@ -631,20 +631,11 @@ define void @testLowerToSHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture
}
define void @testLowerToURHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD8b:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.8b v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD8b:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.8h v2, #1
-; CHECK-GI-NEXT: uaddl.8h v0, v0, v1
-; CHECK-GI-NEXT: add.8h v0, v0, v2
-; CHECK-GI-NEXT: shrn.8b v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD8b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.8b v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i8> %src1 to <8 x i16>
%zextsrc2 = zext <8 x i8> %src2 to <8 x i16>
%add1 = add nuw nsw <8 x i16> %zextsrc1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
@@ -656,20 +647,11 @@ define void @testLowerToURHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture w
}
define void @testLowerToURHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD4h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD4h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.4s v2, #1
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: add.4s v0, v0, v2
-; CHECK-GI-NEXT: shrn.4h v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD4h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.4h v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
%add1 = add nuw nsw <4 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1>
@@ -681,21 +663,11 @@ define void @testLowerToURHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture
}
define void @testLowerToURHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD2s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD2s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI38_0
-; CHECK-GI-NEXT: uaddl.2d v0, v0, v1
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI38_0]
-; CHECK-GI-NEXT: add.2d v0, v0, v1
-; CHECK-GI-NEXT: shrn.2s v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD2s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i32> %src1 to <2 x i64>
%zextsrc2 = zext <2 x i32> %src2 to <2 x i64>
%add1 = add nuw nsw <2 x i64> %zextsrc1, <i64 1, i64 1>
@@ -707,23 +679,11 @@ define void @testLowerToURHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture
}
define void @testLowerToURHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD16b:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.16b v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD16b:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.8h v2, #1
-; CHECK-GI-NEXT: uaddl.8h v3, v0, v1
-; CHECK-GI-NEXT: uaddl2.8h v0, v0, v1
-; CHECK-GI-NEXT: add.8h v1, v3, v2
-; CHECK-GI-NEXT: add.8h v0, v0, v2
-; CHECK-GI-NEXT: shrn.8b v1, v1, #1
-; CHECK-GI-NEXT: shrn2.16b v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD16b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.16b v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <16 x i8> %src1 to <16 x i16>
%zextsrc2 = zext <16 x i8> %src2 to <16 x i16>
%add1 = add nuw nsw <16 x i16> %zextsrc1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
@@ -735,23 +695,11 @@ define void @testLowerToURHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocaptur
}
define void @testLowerToURHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD8h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.8h v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD8h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.4s v2, #1
-; CHECK-GI-NEXT: uaddl.4s v3, v0, v1
-; CHECK-GI-NEXT: uaddl2.4s v0, v0, v1
-; CHECK-GI-NEXT: add.4s v1, v3, v2
-; CHECK-GI-NEXT: add.4s v0, v0, v2
-; CHECK-GI-NEXT: shrn.4h v1, v1, #1
-; CHECK-GI-NEXT: shrn2.8h v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD8h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.8h v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%zextsrc2 = zext <8 x i16> %src2 to <8 x i32>
%add1 = add nuw nsw <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -763,24 +711,11 @@ define void @testLowerToURHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture
}
define void @testLowerToURHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD4s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.4s v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD4s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI41_0
-; CHECK-GI-NEXT: uaddl.2d v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.2d v0, v0, v1
-; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI41_0]
-; CHECK-GI-NEXT: add.2d v1, v2, v3
-; CHECK-GI-NEXT: add.2d v0, v0, v3
-; CHECK-GI-NEXT: shrn.2s v1, v1, #1
-; CHECK-GI-NEXT: shrn2.4s v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD4s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.4s v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
%add1 = add nuw nsw <4 x i64> %zextsrc1, <i64 1, i64 1, i64 1, i64 1>
@@ -792,18 +727,11 @@ define void @testLowerToURHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture
}
define void @testLowerToUHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD8b:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.8b v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD8b:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.8h v0, v0, v1
-; CHECK-GI-NEXT: shrn.8b v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD8b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.8b v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i8> %src1 to <8 x i16>
%zextsrc2 = zext <8 x i8> %src2 to <8 x i16>
%add = add nuw nsw <8 x i16> %zextsrc1, %zextsrc2
@@ -814,18 +742,11 @@ define void @testLowerToUHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture wr
}
define void @testLowerToUHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD4h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD4h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: shrn.4h v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD4h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4h v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
%add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2
@@ -836,18 +757,11 @@ define void @testLowerToUHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture
}
define void @testLowerToUHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD2s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD2s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v0, v0, v1
-; CHECK-GI-NEXT: shrn.2s v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD2s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i32> %src1 to <2 x i64>
%zextsrc2 = zext <2 x i32> %src2 to <2 x i64>
%add = add nuw nsw <2 x i64> %zextsrc1, %zextsrc2
@@ -858,20 +772,11 @@ define void @testLowerToUHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture
}
define void @testLowerToUHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD16b:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.16b v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD16b:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.8h v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.8h v0, v0, v1
-; CHECK-GI-NEXT: shrn.8b v1, v2, #1
-; CHECK-GI-NEXT: shrn2.16b v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD16b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.16b v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <16 x i8> %src1 to <16 x i16>
%zextsrc2 = zext <16 x i8> %src2 to <16 x i16>
%add = add nuw nsw <16 x i16> %zextsrc1, %zextsrc2
@@ -882,20 +787,11 @@ define void @testLowerToUHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture
}
define void @testLowerToUHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD8h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.8h v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD8h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.4s v0, v0, v1
-; CHECK-GI-NEXT: shrn.4h v1, v2, #1
-; CHECK-GI-NEXT: shrn2.8h v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD8h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.8h v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%zextsrc2 = zext <8 x i16> %src2 to <8 x i32>
%add = add nuw nsw <8 x i32> %zextsrc1, %zextsrc2
@@ -906,20 +802,11 @@ define void @testLowerToUHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture
}
define void @testLowerToUHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD4s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4s v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD4s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.2d v0, v0, v1
-; CHECK-GI-NEXT: shrn.2s v1, v2, #1
-; CHECK-GI-NEXT: shrn2.4s v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD4s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4s v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
%add = add nuw nsw <4 x i64> %zextsrc1, %zextsrc2
@@ -930,17 +817,11 @@ define void @testLowerToUHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture
}
define <4 x i32> @hadd16_sext_asr(<4 x i16> %src1, <4 x i16> %src2) {
-; CHECK-SD-LABEL: hadd16_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shadd.4h v0, v0, v1
-; CHECK-SD-NEXT: sshll.4s v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd16_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: saddl.4s v0, v0, v1
-; CHECK-GI-NEXT: sshr.4s v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd16_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shadd.4h v0, v0, v1
+; CHECK-NEXT: sshll.4s v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = sext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = sext <4 x i16> %src2 to <4 x i32>
%add = add nsw <4 x i32> %zextsrc1, %zextsrc2
@@ -949,17 +830,11 @@ define <4 x i32> @hadd16_sext_asr(<4 x i16> %src1, <4 x i16> %src2) {
}
define <4 x i32> @hadd16_zext_asr(<4 x i16> %src1, <4 x i16> %src2) {
-; CHECK-SD-LABEL: hadd16_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: ushll.4s v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd16_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: ushr.4s v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd16_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4h v0, v0, v1
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
%add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2
@@ -981,17 +856,11 @@ define <4 x i32> @hadd16_sext_lsr(<4 x i16> %src1, <4 x i16> %src2) {
}
define <4 x i32> @hadd16_zext_lsr(<4 x i16> %src1, <4 x i16> %src2) {
-; CHECK-SD-LABEL: hadd16_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: ushll.4s v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd16_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: ushr.4s v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd16_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4h v0, v0, v1
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
%add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2
@@ -1009,10 +878,9 @@ define <4 x i64> @hadd32_sext_asr(<4 x i32> %src1, <4 x i32> %src2) {
;
; CHECK-GI-LABEL: hadd32_sext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: saddl.2d v2, v0, v1
-; CHECK-GI-NEXT: saddl2.2d v1, v0, v1
-; CHECK-GI-NEXT: sshr.2d v0, v2, #1
-; CHECK-GI-NEXT: sshr.2d v1, v1, #1
+; CHECK-GI-NEXT: shadd.4s v1, v0, v1
+; CHECK-GI-NEXT: sshll.2d v0, v1, #0
+; CHECK-GI-NEXT: sshll2.2d v1, v1, #0
; CHECK-GI-NEXT: ret
%zextsrc1 = sext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = sext <4 x i32> %src2 to <4 x i64>
@@ -1031,10 +899,9 @@ define <4 x i64> @hadd32_zext_asr(<4 x i32> %src1, <4 x i32> %src2) {
;
; CHECK-GI-LABEL: hadd32_zext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.2d v1, v0, v1
-; CHECK-GI-NEXT: ushr.2d v0, v2, #1
-; CHECK-GI-NEXT: ushr.2d v1, v1, #1
+; CHECK-GI-NEXT: uhadd.4s v1, v0, v1
+; CHECK-GI-NEXT: ushll.2d v0, v1, #0
+; CHECK-GI-NEXT: ushll2.2d v1, v1, #0
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
@@ -1076,10 +943,9 @@ define <4 x i64> @hadd32_zext_lsr(<4 x i32> %src1, <4 x i32> %src2) {
;
; CHECK-GI-LABEL: hadd32_zext_lsr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.2d v1, v0, v1
-; CHECK-GI-NEXT: ushr.2d v0, v2, #1
-; CHECK-GI-NEXT: ushr.2d v1, v1, #1
+; CHECK-GI-NEXT: uhadd.4s v1, v0, v1
+; CHECK-GI-NEXT: ushll.2d v0, v1, #0
+; CHECK-GI-NEXT: ushll2.2d v1, v1, #0
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
@@ -1100,11 +966,7 @@ define <4 x i16> @hadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
;
; CHECK-GI-LABEL: hadd8_sext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.4h v1, v1, #8
-; CHECK-GI-NEXT: shl.4h v0, v0, #8
-; CHECK-GI-NEXT: sshr.4h v1, v1, #8
-; CHECK-GI-NEXT: ssra.4h v1, v0, #8
-; CHECK-GI-NEXT: sshr.4h v0, v1, #1
+; CHECK-GI-NEXT: shadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = sext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = sext <4 x i8> %src2 to <4 x i16>
@@ -1114,21 +976,12 @@ define <4 x i16> @hadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <4 x i16> @hadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: bic.4h v1, #255, lsl #8
-; CHECK-SD-NEXT: bic.4h v0, #255, lsl #8
-; CHECK-SD-NEXT: uhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic.4h v1, #255, lsl #8
+; CHECK-NEXT: bic.4h v0, #255, lsl #8
+; CHECK-NEXT: uhadd.4h v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
%add = add nuw nsw <4 x i16> %zextsrc1, %zextsrc2
@@ -1162,21 +1015,12 @@ define <4 x i16> @hadd8_sext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <4 x i16> @hadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: bic.4h v1, #255, lsl #8
-; CHECK-SD-NEXT: bic.4h v0, #255, lsl #8
-; CHECK-SD-NEXT: uhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic.4h v1, #255, lsl #8
+; CHECK-NEXT: bic.4h v0, #255, lsl #8
+; CHECK-NEXT: uhadd.4h v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
%add = add nuw nsw <4 x i16> %zextsrc1, %zextsrc2
@@ -1185,30 +1029,14 @@ define <4 x i16> @hadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shl.2s v1, v1, #24
-; CHECK-SD-NEXT: shl.2s v0, v0, #24
-; CHECK-SD-NEXT: sshr.2s v1, v1, #24
-; CHECK-SD-NEXT: sshr.2s v0, v0, #24
-; CHECK-SD-NEXT: shadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.2s v1, v1, #24
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: shl.2s v0, v0, #24
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: sshr.2s v1, v1, #24
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: ssra.2s v1, v0, #24
-; CHECK-GI-NEXT: uzp1.4h v0, v1, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: sshl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl.2s v1, v1, #24
+; CHECK-NEXT: shl.2s v0, v0, #24
+; CHECK-NEXT: sshr.2s v1, v1, #24
+; CHECK-NEXT: sshr.2s v0, v0, #24
+; CHECK-NEXT: shadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
%add = add nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1217,29 +1045,13 @@ define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @hadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1282,29 +1094,13 @@ define <2 x i16> @hadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @hadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1324,13 +1120,7 @@ define <4 x i16> @rhadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
;
; CHECK-GI-LABEL: rhadd8_sext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.4h v1, v1, #8
-; CHECK-GI-NEXT: shl.4h v0, v0, #8
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: sshr.4h v1, v1, #8
-; CHECK-GI-NEXT: ssra.4h v1, v0, #8
-; CHECK-GI-NEXT: add.4h v0, v1, v2
-; CHECK-GI-NEXT: sshr.4h v0, v0, #1
+; CHECK-GI-NEXT: srhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = sext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = sext <4 x i8> %src2 to <4 x i16>
@@ -1341,23 +1131,12 @@ define <4 x i16> @rhadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <4 x i16> @rhadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: bic.4h v1, #255, lsl #8
-; CHECK-SD-NEXT: bic.4h v0, #255, lsl #8
-; CHECK-SD-NEXT: urhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: add.4h v0, v0, v2
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic.4h v1, #255, lsl #8
+; CHECK-NEXT: bic.4h v0, #255, lsl #8
+; CHECK-NEXT: urhadd.4h v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
%add = add nuw nsw <4 x i16> %zextsrc1, %zextsrc2
@@ -1397,23 +1176,12 @@ define <4 x i16> @rhadd8_sext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <4 x i16> @rhadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: bic.4h v1, #255, lsl #8
-; CHECK-SD-NEXT: bic.4h v0, #255, lsl #8
-; CHECK-SD-NEXT: urhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: add.4h v0, v0, v2
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic.4h v1, #255, lsl #8
+; CHECK-NEXT: bic.4h v0, #255, lsl #8
+; CHECK-NEXT: urhadd.4h v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
%add = add nuw nsw <4 x i16> %zextsrc1, %zextsrc2
@@ -1423,32 +1191,14 @@ define <4 x i16> @rhadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shl.2s v1, v1, #24
-; CHECK-SD-NEXT: shl.2s v0, v0, #24
-; CHECK-SD-NEXT: sshr.2s v1, v1, #24
-; CHECK-SD-NEXT: sshr.2s v0, v0, #24
-; CHECK-SD-NEXT: srhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.2s v1, v1, #24
-; CHECK-GI-NEXT: shl.2s v0, v0, #24
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: sshr.2s v1, v1, #24
-; CHECK-GI-NEXT: ssra.2s v1, v0, #24
-; CHECK-GI-NEXT: fmov s0, w8
-; CHECK-GI-NEXT: mov.h v0[1], w8
-; CHECK-GI-NEXT: add.2s v1, v1, v2
-; CHECK-GI-NEXT: uzp1.4h v1, v1, v0
-; CHECK-GI-NEXT: neg.4h v0, v0
-; CHECK-GI-NEXT: sshl.4h v0, v1, v0
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl.2s v1, v1, #24
+; CHECK-NEXT: shl.2s v0, v0, #24
+; CHECK-NEXT: sshr.2s v1, v1, #24
+; CHECK-NEXT: sshr.2s v0, v0, #24
+; CHECK-NEXT: srhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
%add = add nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1458,31 +1208,13 @@ define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: fmov s1, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v2
-; CHECK-GI-NEXT: mov.h v1[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v1
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1531,31 +1263,13 @@ define <2 x i16> @rhadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: fmov s1, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v2
-; CHECK-GI-NEXT: mov.h v1[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v1
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
More information about the llvm-commits
mailing list