[llvm] [AArch64][GlobalISel] Add pre-legalizer combines for AVGFLOOR and AVGCEIL (PR #192866)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 22 14:49:53 PDT 2026
https://github.com/Atafid updated https://github.com/llvm/llvm-project/pull/192866
>From a6517d185648b136b889787f0f5c5c1c8610a47b Mon Sep 17 00:00:00 2001
From: Guillaume Di Fatta <difatta.guillaume at gmail.com>
Date: Sun, 19 Apr 2026 21:04:20 +0200
Subject: [PATCH] [GlobalISel] Add pre-legalizer combines for AVGFLOOR and
AVGCEIL
This patch adds GlobalISel pre-legalizer combines to pattern-match and
optimize average operations, bringing GlobalISel on par with SelectionDAG.
Specifically, it matches:
- `(a + b) >> 1` into `G_UAVGFLOOR` / `G_SAVGFLOOR`
- `(a + b + 1) >> 1` into `G_UAVGCEIL` / `G_SAVGCEIL`
Support is included for both scalar and vector types, correctly handling
constants and splat vectors via `isOneOrOneSplat()`. This builds upon
the generic opcodes introduced for AArch64 intrinsics lowering and
enables optimal emission of Neon instructions (e.g., `urhadd`, `shadd`)
directly from generic IR.
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 14 +
.../include/llvm/Target/GlobalISel/Combine.td | 57 +-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 135 +++++
.../AArch64/GlobalISel/combine-avg.mir | 338 ++++++++++++
llvm/test/CodeGen/AArch64/arm64-vhadd.ll | 506 ++++++------------
5 files changed, 698 insertions(+), 352 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 365bbaacfe055..eb890d698b0be 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -1054,6 +1054,20 @@ class CombinerHelper {
// (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
bool matchCtls(MachineInstr &CtlzMI, BuildFnTy &MatchInfo) const;
+ // shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
+ bool matchAVGFloor(MachineInstr &MI, Register X, Register Y,
+ Register ShiftAmtReg, bool IsSigned,
+ MachineRegisterInfo &MRI, BuildFnTy &MatchInfo) const;
+
+ // shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
+ bool matchAVGCeil(MachineInstr &MI, Register Ext1, Register Ext2,
+ Register ShiftAmtReg, Register OneAmtReg, bool IsSigned,
+ MachineRegisterInfo &MRI, BuildFnTy &MatchInfo) const;
+
+ void applyAVG(MachineIRBuilder &B, Register Dst, Register X, Register Y,
+ LLT XTy, LLT DstTy, bool IsSigned, bool NeedExt,
+ bool IsCeil) const;
+
private:
/// Checks for legality of an indexed variant of \p LdSt.
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index df3955909fe85..424bf8884b484 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -2287,6 +2287,60 @@ def overflow_combines: GICombineGroup<[
match_subo_no_overflow
]>;
+def any_sext: GICombinePatFrag<
+ (outs root:$root),
+ (ins $src),
+ [
+ (pattern (G_SEXT $root, $src)),
+ (pattern (G_SEXT_INREG $root, $src, $imm))
+ ]
+>;
+
+def avgfloor_u_match : GICombineRule<
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (G_ZEXT $ext1, $x),
+ (G_ZEXT $ext2, $y),
+ (G_ADD $sum, $ext1, $ext2),
+ (G_LSHR $dst, $sum, $shift):$mi,
+ [{ return Helper.matchAVGFloor(*${mi}, ${x}.getReg(), ${y}.getReg(), ${shift}.getReg(), /*IsSigned=*/false, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
+>;
+
+def avgfloor_s_match : GICombineRule<
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (any_sext $ext1, $x),
+ (any_sext $ext2, $y),
+ (G_ADD $sum, $ext1, $ext2),
+ (G_ASHR $dst, $sum, $shift):$mi,
+ [{ return Helper.matchAVGFloor(*${mi}, ${x}.getReg(), ${y}.getReg(), ${shift}.getReg(), /*IsSigned=*/true, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
+>;
+
+def avgceil_sum : GICombinePatFrag<
+ (outs root:$root),
+ (ins $ext1, $ext2, $one),
+ [
+ (pattern (G_ADD $inner, $ext1, $ext2), (G_ADD $root, $inner, $one)),
+ (pattern (G_ADD $inner, $ext1, $one), (G_ADD $root, $inner, $ext2))
+ ]
+>;
+
+def avgceil_u_match : GICombineRule<
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (avgceil_sum $sum, $ext1, $ext2, $one),
+ (G_LSHR $dst, $sum, $shift):$mi,
+ [{ return Helper.matchAVGCeil(*${mi}, ${ext1}.getReg(), ${ext2}.getReg(), ${shift}.getReg(), ${one}.getReg(), /*IsSigned=*/false, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
+>;
+
+def avgceil_s_match : GICombineRule<
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (avgceil_sum $sum, $ext1, $ext2, $one),
+ (G_ASHR $dst, $sum, $shift):$mi,
+ [{ return Helper.matchAVGCeil(*${mi}, ${ext1}.getReg(), ${ext2}.getReg(), ${shift}.getReg(), ${one}.getReg(), /*IsSigned=*/true, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
+>;
+
// FIXME: These should use the custom predicate feature once it lands.
def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
undef_to_negative_one,
@@ -2376,7 +2430,8 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
combine_use_vector_truncate, merge_combines, overflow_combines,
truncsat_combines, lshr_of_trunc_of_lshr, ctls_combines, add_shift, sub_one_from_sub,
- binop_with_neg, sub_minus_one]>;
+ binop_with_neg, sub_minus_one, avgfloor_u_match, avgfloor_s_match,
+ avgceil_u_match, avgceil_s_match]>;
// A combine group used to for prelegalizer combiners at -O0. The combines in
// this group have been selected based on experiments to balance code size and
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 0a7d682c44049..7a1c825b6eac6 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -8692,3 +8692,138 @@ bool CombinerHelper::matchCtls(MachineInstr &CtlzMI,
return true;
}
+
+static bool matchAnySExt(MachineRegisterInfo &MRI, Register Reg,
+ Register &ExtSrc) {
+ if (mi_match(Reg, MRI, m_GSExt(m_Reg(ExtSrc))))
+ return true;
+
+ MachineInstr *ExtInst = MRI.getVRegDef(Reg);
+ if (ExtInst && ExtInst->getOpcode() == TargetOpcode::G_SEXT_INREG) {
+ ExtSrc = ExtInst->getOperand(1).getReg();
+ return true;
+ }
+ return false;
+}
+
+// Fold shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
+bool CombinerHelper::matchAVGFloor(MachineInstr &MI, Register X, Register Y,
+ Register ShiftAmtReg, bool IsSigned,
+ MachineRegisterInfo &MRI,
+ BuildFnTy &MatchInfo) const {
+ assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
+ MI.getOpcode() == TargetOpcode::G_ASHR) &&
+ "Expected G_LSHR/G_ASHR");
+
+ if (!isOneOrOneSplat(ShiftAmtReg, false))
+ return false;
+
+ LLT XTy = MRI.getType(X);
+ LLT YTy = MRI.getType(Y);
+
+ if (XTy != YTy)
+ return false;
+
+ Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
+
+ if (!DstTy.isVector())
+ return false;
+
+ bool NeedExt = DstTy.getSizeInBits() > XTy.getSizeInBits();
+
+ MatchInfo = [=](MachineIRBuilder &B) {
+ CombinerHelper::applyAVG(B, Dst, X, Y, XTy, DstTy, IsSigned, NeedExt,
+ false);
+ };
+
+ return true;
+}
+
+// Fold shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
+bool CombinerHelper::matchAVGCeil(MachineInstr &MI, Register Ext1,
+ Register Ext2, Register ShiftAmtReg,
+ Register OneAmtReg, bool IsSigned,
+ MachineRegisterInfo &MRI,
+ BuildFnTy &MatchInfo) const {
+ assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
+ MI.getOpcode() == TargetOpcode::G_ASHR) &&
+ "Expected G_LSHR/G_ASHR");
+
+ if (!isOneOrOneSplat(ShiftAmtReg, false) ||
+ !isOneOrOneSplat(OneAmtReg, false))
+ return false;
+
+ Register X, Y;
+ if (IsSigned) {
+ if (!matchAnySExt(MRI, Ext1, X) || !matchAnySExt(MRI, Ext2, Y)) {
+ return false;
+ }
+ } else {
+ if (!mi_match(Ext1, MRI, m_GZExt(m_Reg(X))) ||
+ !mi_match(Ext2, MRI, m_GZExt(m_Reg(Y))))
+ return false;
+ }
+
+ LLT XTy = MRI.getType(X);
+ LLT YTy = MRI.getType(Y);
+
+ if (XTy != YTy)
+ return false;
+
+ Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
+
+ if (!DstTy.isVector())
+ return false;
+
+ bool NeedExt = DstTy.getSizeInBits() > XTy.getSizeInBits();
+
+ MatchInfo = [=](MachineIRBuilder &B) {
+ CombinerHelper::applyAVG(B, Dst, X, Y, XTy, DstTy, IsSigned, NeedExt, true);
+ };
+
+ return true;
+}
+
+void CombinerHelper::applyAVG(MachineIRBuilder &B, Register Dst, Register X,
+ Register Y, LLT XTy, LLT DstTy, bool IsSigned,
+ bool NeedExt, bool IsCeil) const {
+ unsigned TargetOpc;
+ if (IsCeil)
+ TargetOpc = IsSigned ? TargetOpcode::G_SAVGCEIL : TargetOpcode::G_UAVGCEIL;
+ else
+ TargetOpc =
+ IsSigned ? TargetOpcode::G_SAVGFLOOR : TargetOpcode::G_UAVGFLOOR;
+
+ if (!NeedExt) {
+ B.buildInstr(TargetOpc, {Dst}, {X, Y});
+ return;
+ }
+
+ if (DstTy.isVector()) {
+ Register ExtX = B.buildAnyExt(DstTy, X).getReg(0);
+ Register ExtY = B.buildAnyExt(DstTy, Y).getReg(0);
+ unsigned NarrowSize = XTy.getScalarSizeInBits();
+
+ Register PromotedX, PromotedY;
+ if (IsSigned) {
+ PromotedX = B.buildSExtInReg(DstTy, ExtX, NarrowSize).getReg(0);
+ PromotedY = B.buildSExtInReg(DstTy, ExtY, NarrowSize).getReg(0);
+ } else {
+ auto Mask = B.buildConstant(
+ DstTy, APInt::getLowBitsSet(DstTy.getScalarSizeInBits(), NarrowSize));
+ PromotedX = B.buildAnd(DstTy, ExtX, Mask).getReg(0);
+ PromotedY = B.buildAnd(DstTy, ExtY, Mask).getReg(0);
+ }
+
+ B.buildInstr(TargetOpc, {Dst}, {PromotedX, PromotedY});
+ return;
+ }
+
+ auto Avg = B.buildInstr(TargetOpc, {XTy}, {X, Y});
+ if (IsSigned)
+ B.buildSExt(Dst, Avg);
+ else
+ B.buildZExt(Dst, Avg);
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
new file mode 100644
index 0000000000000..3e6b99f86a84e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
@@ -0,0 +1,338 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner %s -o - | FileCheck %s
+
+...
+---
+name: test_uavgfloor_i16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_uavgfloor_i16
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[UAVGFLOOR:%[0-9]+]]:_(s16) = G_UAVGFLOOR [[TRUNC]], [[TRUNC1]]
+ ; CHECK-NEXT: %ret_ext:_(s32) = G_ANYEXT [[UAVGFLOOR]](s16)
+ ; CHECK-NEXT: $w0 = COPY %ret_ext(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 1
+ %6:_(s32) = G_LSHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgfloor_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_uavgfloor_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<8 x s32>) = G_ANYEXT [[COPY]](<8 x s16>)
+ ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(<8 x s32>) = G_ANYEXT [[COPY1]](<8 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<8 x s32>) = G_AND [[ANYEXT]], [[BUILD_VECTOR]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<8 x s32>) = G_AND [[ANYEXT1]], [[BUILD_VECTOR]]
+ ; CHECK-NEXT: %shift:_(<8 x s32>) = G_UAVGFLOOR [[AND]], [[AND1]]
+ ; CHECK-NEXT: %res:_(<8 x s16>) = G_TRUNC %shift(<8 x s32>)
+ ; CHECK-NEXT: $q0 = COPY %res(<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+
+ %ext0:_(<8 x s32>) = G_ZEXT %0
+ %ext1:_(<8 x s32>) = G_ZEXT %1
+
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_LSHR %sum, %v_shift_cst
+
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_savgfloor_i16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_savgfloor_i16
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[SAVGFLOOR:%[0-9]+]]:_(s32) = G_SAVGFLOOR %w0_copy, %w1_copy
+ ; CHECK-NEXT: $w0 = COPY [[SAVGFLOOR]](s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_SEXT %0(s16)
+ %3:_(s32) = G_SEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 1
+ %6:_(s32) = G_ASHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_savgfloor_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_savgfloor_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<8 x s32>) = G_ANYEXT [[COPY]](<8 x s16>)
+ ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(<8 x s32>) = G_ANYEXT [[COPY1]](<8 x s16>)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<8 x s32>) = G_SEXT_INREG [[ANYEXT]], 16
+ ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(<8 x s32>) = G_SEXT_INREG [[ANYEXT1]], 16
+ ; CHECK-NEXT: %shift:_(<8 x s32>) = G_SAVGFLOOR [[SEXT_INREG]], [[SEXT_INREG1]]
+ ; CHECK-NEXT: %res:_(<8 x s16>) = G_TRUNC %shift(<8 x s32>)
+ ; CHECK-NEXT: $q0 = COPY %res(<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+
+ %ext0:_(<8 x s32>) = G_SEXT %0
+ %ext1:_(<8 x s32>) = G_SEXT %1
+
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_ASHR %sum, %v_shift_cst
+
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_negative_floor
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_negative_floor
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+ ; CHECK-NEXT: $w0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 2
+ %6:_(s32) = G_LSHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgceil_s32
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_uavgceil_s32
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[UAVGCEIL:%[0-9]+]]:_(s16) = G_UAVGCEIL [[TRUNC]], [[TRUNC1]]
+ ; CHECK-NEXT: %res:_(s32) = G_ZEXT [[UAVGCEIL]](s16)
+ ; CHECK-NEXT: $w0 = COPY %res(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %sum:_(s32) = G_ADD %2, %3
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %add_one:_(s32) = G_ADD %sum, %cst1
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_LSHR %add_one, %cst_shift
+
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_savgceil_s32_associative
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_savgceil_s32_associative
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: %res:_(s32) = G_SAVGCEIL %w0_copy, %w1_copy
+ ; CHECK-NEXT: $w0 = COPY %res(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_SEXT %0(s16)
+ %3:_(s32) = G_SEXT %1(s16)
+
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %add_one:_(s32) = G_ADD %2, %cst1
+ %sum:_(s32) = G_ADD %add_one, %3
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_ASHR %sum, %cst_shift
+
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgceil_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_uavgceil_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<8 x s32>) = G_ANYEXT [[COPY]](<8 x s16>)
+ ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(<8 x s32>) = G_ANYEXT [[COPY1]](<8 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<8 x s32>) = G_AND [[ANYEXT]], [[BUILD_VECTOR]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<8 x s32>) = G_AND [[ANYEXT1]], [[BUILD_VECTOR]]
+ ; CHECK-NEXT: %shift:_(<8 x s32>) = G_UAVGCEIL [[AND]], [[AND1]]
+ ; CHECK-NEXT: %res:_(<8 x s16>) = G_TRUNC %shift(<8 x s32>)
+ ; CHECK-NEXT: $q0 = COPY %res(<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+ %ext0:_(<8 x s32>) = G_ZEXT %0
+ %ext1:_(<8 x s32>) = G_ZEXT %1
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %v_cst1:_(<8 x s32>) = G_BUILD_VECTOR %cst1, %cst1, %cst1, %cst1, %cst1, %cst1, %cst1, %cst1
+ %add_one:_(<8 x s32>) = G_ADD %sum, %v_cst1
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_LSHR %add_one, %v_shift_cst
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_negative_ceil_bad_constant
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_negative_ceil_bad_constant
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: %sum:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: %cst2:_(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: %add_two:_(s32) = G_ADD %sum, %cst2
+ ; CHECK-NEXT: %cst_shift:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %res:_(s32) = G_LSHR %add_two, %cst_shift(s32)
+ ; CHECK-NEXT: $w0 = COPY %res(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+ %sum:_(s32) = G_ADD %2, %3
+ %cst2:_(s32) = G_CONSTANT i32 2
+ %add_two:_(s32) = G_ADD %sum, %cst2
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_LSHR %add_two, %cst_shift
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
index 4723867bc99f0..95c918f695066 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
@@ -639,10 +639,13 @@ define void @testLowerToURHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture w
;
; CHECK-GI-LABEL: testLowerToURHADD8b:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.8h v2, #1
-; CHECK-GI-NEXT: uaddl.8h v0, v0, v1
-; CHECK-GI-NEXT: add.8h v0, v0, v2
-; CHECK-GI-NEXT: shrn.8b v0, v0, #1
+; CHECK-GI-NEXT: movi.2d v2, #0xff00ff00ff00ff
+; CHECK-GI-NEXT: ushll.8h v0, v0, #0
+; CHECK-GI-NEXT: ushll.8h v1, v1, #0
+; CHECK-GI-NEXT: and.16b v0, v0, v2
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: urhadd.8h v0, v0, v1
+; CHECK-GI-NEXT: xtn.8b v0, v0
; CHECK-GI-NEXT: str d0, [x0]
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <8 x i8> %src1 to <8 x i16>
@@ -664,10 +667,13 @@ define void @testLowerToURHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture
;
; CHECK-GI-LABEL: testLowerToURHADD4h:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.4s v2, #1
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: add.4s v0, v0, v2
-; CHECK-GI-NEXT: shrn.4h v0, v0, #1
+; CHECK-GI-NEXT: movi.2d v2, #0x00ffff0000ffff
+; CHECK-GI-NEXT: ushll.4s v0, v0, #0
+; CHECK-GI-NEXT: ushll.4s v1, v1, #0
+; CHECK-GI-NEXT: and.16b v0, v0, v2
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: urhadd.4s v0, v0, v1
+; CHECK-GI-NEXT: xtn.4h v0, v0
; CHECK-GI-NEXT: str d0, [x0]
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
@@ -681,21 +687,11 @@ define void @testLowerToURHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture
}
define void @testLowerToURHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD2s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD2s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI38_0
-; CHECK-GI-NEXT: uaddl.2d v0, v0, v1
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI38_0]
-; CHECK-GI-NEXT: add.2d v0, v0, v1
-; CHECK-GI-NEXT: shrn.2s v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD2s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i32> %src1 to <2 x i64>
%zextsrc2 = zext <2 x i32> %src2 to <2 x i64>
%add1 = add nuw nsw <2 x i64> %zextsrc1, <i64 1, i64 1>
@@ -707,23 +703,11 @@ define void @testLowerToURHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture
}
define void @testLowerToURHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD16b:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.16b v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD16b:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.8h v2, #1
-; CHECK-GI-NEXT: uaddl.8h v3, v0, v1
-; CHECK-GI-NEXT: uaddl2.8h v0, v0, v1
-; CHECK-GI-NEXT: add.8h v1, v3, v2
-; CHECK-GI-NEXT: add.8h v0, v0, v2
-; CHECK-GI-NEXT: shrn.8b v1, v1, #1
-; CHECK-GI-NEXT: shrn2.16b v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD16b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.16b v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <16 x i8> %src1 to <16 x i16>
%zextsrc2 = zext <16 x i8> %src2 to <16 x i16>
%add1 = add nuw nsw <16 x i16> %zextsrc1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
@@ -735,23 +719,11 @@ define void @testLowerToURHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocaptur
}
define void @testLowerToURHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD8h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.8h v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD8h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.4s v2, #1
-; CHECK-GI-NEXT: uaddl.4s v3, v0, v1
-; CHECK-GI-NEXT: uaddl2.4s v0, v0, v1
-; CHECK-GI-NEXT: add.4s v1, v3, v2
-; CHECK-GI-NEXT: add.4s v0, v0, v2
-; CHECK-GI-NEXT: shrn.4h v1, v1, #1
-; CHECK-GI-NEXT: shrn2.8h v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD8h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.8h v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%zextsrc2 = zext <8 x i16> %src2 to <8 x i32>
%add1 = add nuw nsw <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -763,24 +735,11 @@ define void @testLowerToURHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture
}
define void @testLowerToURHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD4s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.4s v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD4s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI41_0
-; CHECK-GI-NEXT: uaddl.2d v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.2d v0, v0, v1
-; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI41_0]
-; CHECK-GI-NEXT: add.2d v1, v2, v3
-; CHECK-GI-NEXT: add.2d v0, v0, v3
-; CHECK-GI-NEXT: shrn.2s v1, v1, #1
-; CHECK-GI-NEXT: shrn2.4s v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD4s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.4s v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
%add1 = add nuw nsw <4 x i64> %zextsrc1, <i64 1, i64 1, i64 1, i64 1>
@@ -800,8 +759,13 @@ define void @testLowerToUHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture wr
;
; CHECK-GI-LABEL: testLowerToUHADD8b:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.8h v0, v0, v1
-; CHECK-GI-NEXT: shrn.8b v0, v0, #1
+; CHECK-GI-NEXT: movi.2d v2, #0xff00ff00ff00ff
+; CHECK-GI-NEXT: ushll.8h v0, v0, #0
+; CHECK-GI-NEXT: ushll.8h v1, v1, #0
+; CHECK-GI-NEXT: and.16b v0, v0, v2
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: uhadd.8h v0, v0, v1
+; CHECK-GI-NEXT: xtn.8b v0, v0
; CHECK-GI-NEXT: str d0, [x0]
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <8 x i8> %src1 to <8 x i16>
@@ -822,8 +786,13 @@ define void @testLowerToUHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture
;
; CHECK-GI-LABEL: testLowerToUHADD4h:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: shrn.4h v0, v0, #1
+; CHECK-GI-NEXT: movi.2d v2, #0x00ffff0000ffff
+; CHECK-GI-NEXT: ushll.4s v0, v0, #0
+; CHECK-GI-NEXT: ushll.4s v1, v1, #0
+; CHECK-GI-NEXT: and.16b v0, v0, v2
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: uhadd.4s v0, v0, v1
+; CHECK-GI-NEXT: xtn.4h v0, v0
; CHECK-GI-NEXT: str d0, [x0]
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
@@ -836,18 +805,11 @@ define void @testLowerToUHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture
}
define void @testLowerToUHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD2s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD2s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v0, v0, v1
-; CHECK-GI-NEXT: shrn.2s v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD2s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i32> %src1 to <2 x i64>
%zextsrc2 = zext <2 x i32> %src2 to <2 x i64>
%add = add nuw nsw <2 x i64> %zextsrc1, %zextsrc2
@@ -858,20 +820,11 @@ define void @testLowerToUHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture
}
define void @testLowerToUHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD16b:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.16b v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD16b:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.8h v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.8h v0, v0, v1
-; CHECK-GI-NEXT: shrn.8b v1, v2, #1
-; CHECK-GI-NEXT: shrn2.16b v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD16b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.16b v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <16 x i8> %src1 to <16 x i16>
%zextsrc2 = zext <16 x i8> %src2 to <16 x i16>
%add = add nuw nsw <16 x i16> %zextsrc1, %zextsrc2
@@ -882,20 +835,11 @@ define void @testLowerToUHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture
}
define void @testLowerToUHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD8h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.8h v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD8h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.4s v0, v0, v1
-; CHECK-GI-NEXT: shrn.4h v1, v2, #1
-; CHECK-GI-NEXT: shrn2.8h v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD8h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.8h v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%zextsrc2 = zext <8 x i16> %src2 to <8 x i32>
%add = add nuw nsw <8 x i32> %zextsrc1, %zextsrc2
@@ -906,20 +850,11 @@ define void @testLowerToUHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture
}
define void @testLowerToUHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD4s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4s v0, v0, v1
-; CHECK-SD-NEXT: str q0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD4s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.2d v0, v0, v1
-; CHECK-GI-NEXT: shrn.2s v1, v2, #1
-; CHECK-GI-NEXT: shrn2.4s v1, v0, #1
-; CHECK-GI-NEXT: str q1, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD4s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4s v0, v0, v1
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
%add = add nuw nsw <4 x i64> %zextsrc1, %zextsrc2
@@ -938,8 +873,13 @@ define <4 x i32> @hadd16_sext_asr(<4 x i16> %src1, <4 x i16> %src2) {
;
; CHECK-GI-LABEL: hadd16_sext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: saddl.4s v0, v0, v1
-; CHECK-GI-NEXT: sshr.4s v0, v0, #1
+; CHECK-GI-NEXT: ushll.4s v0, v0, #0
+; CHECK-GI-NEXT: ushll.4s v1, v1, #0
+; CHECK-GI-NEXT: shl.4s v0, v0, #16
+; CHECK-GI-NEXT: shl.4s v1, v1, #16
+; CHECK-GI-NEXT: sshr.4s v0, v0, #16
+; CHECK-GI-NEXT: sshr.4s v1, v1, #16
+; CHECK-GI-NEXT: shadd.4s v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = sext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = sext <4 x i16> %src2 to <4 x i32>
@@ -957,8 +897,12 @@ define <4 x i32> @hadd16_zext_asr(<4 x i16> %src1, <4 x i16> %src2) {
;
; CHECK-GI-LABEL: hadd16_zext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: ushr.4s v0, v0, #1
+; CHECK-GI-NEXT: movi.2d v2, #0x00ffff0000ffff
+; CHECK-GI-NEXT: ushll.4s v0, v0, #0
+; CHECK-GI-NEXT: ushll.4s v1, v1, #0
+; CHECK-GI-NEXT: and.16b v0, v0, v2
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: uhadd.4s v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
@@ -989,8 +933,12 @@ define <4 x i32> @hadd16_zext_lsr(<4 x i16> %src1, <4 x i16> %src2) {
;
; CHECK-GI-LABEL: hadd16_zext_lsr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: ushr.4s v0, v0, #1
+; CHECK-GI-NEXT: movi.2d v2, #0x00ffff0000ffff
+; CHECK-GI-NEXT: ushll.4s v0, v0, #0
+; CHECK-GI-NEXT: ushll.4s v1, v1, #0
+; CHECK-GI-NEXT: and.16b v0, v0, v2
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: uhadd.4s v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
@@ -1000,20 +948,12 @@ define <4 x i32> @hadd16_zext_lsr(<4 x i16> %src1, <4 x i16> %src2) {
}
define <4 x i64> @hadd32_sext_asr(<4 x i32> %src1, <4 x i32> %src2) {
-; CHECK-SD-LABEL: hadd32_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shadd.4s v0, v0, v1
-; CHECK-SD-NEXT: sshll2.2d v1, v0, #0
-; CHECK-SD-NEXT: sshll.2d v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd32_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: saddl.2d v2, v0, v1
-; CHECK-GI-NEXT: saddl2.2d v1, v0, v1
-; CHECK-GI-NEXT: sshr.2d v0, v2, #1
-; CHECK-GI-NEXT: sshr.2d v1, v1, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd32_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shadd.4s v0, v0, v1
+; CHECK-NEXT: sshll2.2d v1, v0, #0
+; CHECK-NEXT: sshll.2d v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = sext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = sext <4 x i32> %src2 to <4 x i64>
%add = add nsw <4 x i64> %zextsrc1, %zextsrc2
@@ -1022,20 +962,12 @@ define <4 x i64> @hadd32_sext_asr(<4 x i32> %src1, <4 x i32> %src2) {
}
define <4 x i64> @hadd32_zext_asr(<4 x i32> %src1, <4 x i32> %src2) {
-; CHECK-SD-LABEL: hadd32_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4s v0, v0, v1
-; CHECK-SD-NEXT: ushll2.2d v1, v0, #0
-; CHECK-SD-NEXT: ushll.2d v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd32_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.2d v1, v0, v1
-; CHECK-GI-NEXT: ushr.2d v0, v2, #1
-; CHECK-GI-NEXT: ushr.2d v1, v1, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd32_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4s v0, v0, v1
+; CHECK-NEXT: ushll2.2d v1, v0, #0
+; CHECK-NEXT: ushll.2d v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
%add = add nuw nsw <4 x i64> %zextsrc1, %zextsrc2
@@ -1067,20 +999,12 @@ define <4 x i64> @hadd32_sext_lsr(<4 x i32> %src1, <4 x i32> %src2) {
}
define <4 x i64> @hadd32_zext_lsr(<4 x i32> %src1, <4 x i32> %src2) {
-; CHECK-SD-LABEL: hadd32_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4s v0, v0, v1
-; CHECK-SD-NEXT: ushll2.2d v1, v0, #0
-; CHECK-SD-NEXT: ushll.2d v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd32_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v2, v0, v1
-; CHECK-GI-NEXT: uaddl2.2d v1, v0, v1
-; CHECK-GI-NEXT: ushr.2d v0, v2, #1
-; CHECK-GI-NEXT: ushr.2d v1, v1, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd32_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4s v0, v0, v1
+; CHECK-NEXT: ushll2.2d v1, v0, #0
+; CHECK-NEXT: ushll.2d v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
%zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
%add = add nuw nsw <4 x i64> %zextsrc1, %zextsrc2
@@ -1100,11 +1024,7 @@ define <4 x i16> @hadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
;
; CHECK-GI-LABEL: hadd8_sext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.4h v1, v1, #8
-; CHECK-GI-NEXT: shl.4h v0, v0, #8
-; CHECK-GI-NEXT: sshr.4h v1, v1, #8
-; CHECK-GI-NEXT: ssra.4h v1, v0, #8
-; CHECK-GI-NEXT: sshr.4h v0, v1, #1
+; CHECK-GI-NEXT: shadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = sext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = sext <4 x i8> %src2 to <4 x i16>
@@ -1126,8 +1046,7 @@ define <4 x i16> @hadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {
; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT: and.8b v0, v0, v2
; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
+; CHECK-GI-NEXT: uhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -1174,8 +1093,7 @@ define <4 x i16> @hadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT: and.8b v0, v0, v2
; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
+; CHECK-GI-NEXT: uhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -1185,30 +1103,14 @@ define <4 x i16> @hadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shl.2s v1, v1, #24
-; CHECK-SD-NEXT: shl.2s v0, v0, #24
-; CHECK-SD-NEXT: sshr.2s v1, v1, #24
-; CHECK-SD-NEXT: sshr.2s v0, v0, #24
-; CHECK-SD-NEXT: shadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.2s v1, v1, #24
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: shl.2s v0, v0, #24
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: sshr.2s v1, v1, #24
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: ssra.2s v1, v0, #24
-; CHECK-GI-NEXT: uzp1.4h v0, v1, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: sshl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl.2s v1, v1, #24
+; CHECK-NEXT: shl.2s v0, v0, #24
+; CHECK-NEXT: sshr.2s v1, v1, #24
+; CHECK-NEXT: sshr.2s v0, v0, #24
+; CHECK-NEXT: shadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
%add = add nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1217,29 +1119,13 @@ define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @hadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1282,29 +1168,13 @@ define <2 x i16> @hadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @hadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1324,13 +1194,7 @@ define <4 x i16> @rhadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
;
; CHECK-GI-LABEL: rhadd8_sext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.4h v1, v1, #8
-; CHECK-GI-NEXT: shl.4h v0, v0, #8
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: sshr.4h v1, v1, #8
-; CHECK-GI-NEXT: ssra.4h v1, v0, #8
-; CHECK-GI-NEXT: add.4h v0, v1, v2
-; CHECK-GI-NEXT: sshr.4h v0, v0, #1
+; CHECK-GI-NEXT: srhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = sext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = sext <4 x i8> %src2 to <4 x i16>
@@ -1353,10 +1217,7 @@ define <4 x i16> @rhadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {
; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT: and.8b v0, v0, v2
; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: add.4h v0, v0, v2
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
+; CHECK-GI-NEXT: urhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -1409,10 +1270,7 @@ define <4 x i16> @rhadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT: and.8b v0, v0, v2
; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: add.4h v0, v0, v2
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
+; CHECK-GI-NEXT: urhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -1423,32 +1281,14 @@ define <4 x i16> @rhadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shl.2s v1, v1, #24
-; CHECK-SD-NEXT: shl.2s v0, v0, #24
-; CHECK-SD-NEXT: sshr.2s v1, v1, #24
-; CHECK-SD-NEXT: sshr.2s v0, v0, #24
-; CHECK-SD-NEXT: srhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.2s v1, v1, #24
-; CHECK-GI-NEXT: shl.2s v0, v0, #24
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: sshr.2s v1, v1, #24
-; CHECK-GI-NEXT: ssra.2s v1, v0, #24
-; CHECK-GI-NEXT: fmov s0, w8
-; CHECK-GI-NEXT: mov.h v0[1], w8
-; CHECK-GI-NEXT: add.2s v1, v1, v2
-; CHECK-GI-NEXT: uzp1.4h v1, v1, v0
-; CHECK-GI-NEXT: neg.4h v0, v0
-; CHECK-GI-NEXT: sshl.4h v0, v1, v0
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl.2s v1, v1, #24
+; CHECK-NEXT: shl.2s v0, v0, #24
+; CHECK-NEXT: sshr.2s v1, v1, #24
+; CHECK-NEXT: sshr.2s v0, v0, #24
+; CHECK-NEXT: srhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
%add = add nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1458,31 +1298,13 @@ define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: fmov s1, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v2
-; CHECK-GI-NEXT: mov.h v1[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v1
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1531,31 +1353,13 @@ define <2 x i16> @rhadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: fmov s1, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v2
-; CHECK-GI-NEXT: mov.h v1[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v1
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
More information about the llvm-commits
mailing list