[llvm] [AArch64][GlobalISel] Add pre-legalizer combines for AVGFLOOR and AVGCEIL (PR #192866)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 22 16:46:38 PDT 2026
https://github.com/Atafid updated https://github.com/llvm/llvm-project/pull/192866
>From 3b26b429b11c40ff2380e2921b52ea5a270d0237 Mon Sep 17 00:00:00 2001
From: Guillaume Di Fatta <difatta.guillaume at gmail.com>
Date: Sun, 19 Apr 2026 21:04:20 +0200
Subject: [PATCH] [GlobalISel] Add pre-legalizer combines for AVGFLOOR and
AVGCEIL
This patch adds GlobalISel pre-legalizer combines to pattern-match and
optimize average operations, bringing GlobalISel on par with SelectionDAG.
Specifically, it matches:
- `(a + b) >> 1` into `G_UAVGFLOOR` / `G_SAVGFLOOR`
- `(a + b + 1) >> 1` into `G_UAVGCEIL` / `G_SAVGCEIL`
Support is included for both scalar and vector types, correctly handling
constants and splat vectors via `isOneOrOneSplat()`. This builds upon
the generic opcodes introduced for AArch64 intrinsics lowering and
enables optimal emission of Neon instructions (e.g., `urhadd`, `shadd`)
directly from generic IR.
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 14 +
.../include/llvm/Target/GlobalISel/Combine.td | 57 ++-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 135 +++++++
.../AArch64/GlobalISel/combine-avg.mir | 358 +++++++++++++++++
llvm/test/CodeGen/AArch64/arm64-vhadd.ll | 377 +++++-------------
5 files changed, 658 insertions(+), 283 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 365bbaacfe055..eb890d698b0be 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -1054,6 +1054,20 @@ class CombinerHelper {
// (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
bool matchCtls(MachineInstr &CtlzMI, BuildFnTy &MatchInfo) const;
+ // shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
+ bool matchAVGFloor(MachineInstr &MI, Register X, Register Y,
+ Register ShiftAmtReg, bool IsSigned,
+ MachineRegisterInfo &MRI, BuildFnTy &MatchInfo) const;
+
+ // shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
+ bool matchAVGCeil(MachineInstr &MI, Register Ext1, Register Ext2,
+ Register ShiftAmtReg, Register OneAmtReg, bool IsSigned,
+ MachineRegisterInfo &MRI, BuildFnTy &MatchInfo) const;
+
+ void applyAVG(MachineIRBuilder &B, Register Dst, Register X, Register Y,
+ LLT XTy, LLT DstTy, bool IsSigned, bool NeedExt,
+ bool IsCeil) const;
+
private:
/// Checks for legality of an indexed variant of \p LdSt.
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index df3955909fe85..424bf8884b484 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -2287,6 +2287,60 @@ def overflow_combines: GICombineGroup<[
match_subo_no_overflow
]>;
+def any_sext: GICombinePatFrag<
+ (outs root:$root),
+ (ins $src),
+ [
+ (pattern (G_SEXT $root, $src)),
+ (pattern (G_SEXT_INREG $root, $src, $imm))
+ ]
+>;
+
+def avgfloor_u_match : GICombineRule<
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (G_ZEXT $ext1, $x),
+ (G_ZEXT $ext2, $y),
+ (G_ADD $sum, $ext1, $ext2),
+ (G_LSHR $dst, $sum, $shift):$mi,
+ [{ return Helper.matchAVGFloor(*${mi}, ${x}.getReg(), ${y}.getReg(), ${shift}.getReg(), /*IsSigned=*/false, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
+>;
+
+def avgfloor_s_match : GICombineRule<
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (any_sext $ext1, $x),
+ (any_sext $ext2, $y),
+ (G_ADD $sum, $ext1, $ext2),
+ (G_ASHR $dst, $sum, $shift):$mi,
+ [{ return Helper.matchAVGFloor(*${mi}, ${x}.getReg(), ${y}.getReg(), ${shift}.getReg(), /*IsSigned=*/true, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
+>;
+
+def avgceil_sum : GICombinePatFrag<
+ (outs root:$root),
+ (ins $ext1, $ext2, $one),
+ [
+ (pattern (G_ADD $inner, $ext1, $ext2), (G_ADD $root, $inner, $one)),
+ (pattern (G_ADD $inner, $ext1, $one), (G_ADD $root, $inner, $ext2))
+ ]
+>;
+
+def avgceil_u_match : GICombineRule<
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (avgceil_sum $sum, $ext1, $ext2, $one),
+ (G_LSHR $dst, $sum, $shift):$mi,
+ [{ return Helper.matchAVGCeil(*${mi}, ${ext1}.getReg(), ${ext2}.getReg(), ${shift}.getReg(), ${one}.getReg(), /*IsSigned=*/false, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
+>;
+
+def avgceil_s_match : GICombineRule<
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (avgceil_sum $sum, $ext1, $ext2, $one),
+ (G_ASHR $dst, $sum, $shift):$mi,
+ [{ return Helper.matchAVGCeil(*${mi}, ${ext1}.getReg(), ${ext2}.getReg(), ${shift}.getReg(), ${one}.getReg(), /*IsSigned=*/true, MRI, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
+>;
+
// FIXME: These should use the custom predicate feature once it lands.
def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
undef_to_negative_one,
@@ -2376,7 +2430,8 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
combine_use_vector_truncate, merge_combines, overflow_combines,
truncsat_combines, lshr_of_trunc_of_lshr, ctls_combines, add_shift, sub_one_from_sub,
- binop_with_neg, sub_minus_one]>;
+ binop_with_neg, sub_minus_one, avgfloor_u_match, avgfloor_s_match,
+ avgceil_u_match, avgceil_s_match]>;
// A combine group used to for prelegalizer combiners at -O0. The combines in
// this group have been selected based on experiments to balance code size and
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 9a9ba9b6b33f0..3b0e82df64ee2 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -8680,3 +8680,138 @@ bool CombinerHelper::matchCtls(MachineInstr &CtlzMI,
return true;
}
+
+static bool matchAnySExt(MachineRegisterInfo &MRI, Register Reg,
+ Register &ExtSrc) {
+ if (mi_match(Reg, MRI, m_GSExt(m_Reg(ExtSrc))))
+ return true;
+
+ MachineInstr *ExtInst = MRI.getVRegDef(Reg);
+ if (ExtInst && ExtInst->getOpcode() == TargetOpcode::G_SEXT_INREG) {
+ ExtSrc = ExtInst->getOperand(1).getReg();
+ return true;
+ }
+ return false;
+}
+
+// Fold shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
+bool CombinerHelper::matchAVGFloor(MachineInstr &MI, Register X, Register Y,
+ Register ShiftAmtReg, bool IsSigned,
+ MachineRegisterInfo &MRI,
+ BuildFnTy &MatchInfo) const {
+ assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
+ MI.getOpcode() == TargetOpcode::G_ASHR) &&
+ "Expected G_LSHR/G_ASHR");
+
+ if (!isOneOrOneSplat(ShiftAmtReg, false))
+ return false;
+
+ LLT XTy = MRI.getType(X);
+ LLT YTy = MRI.getType(Y);
+
+ if (XTy != YTy)
+ return false;
+
+ Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
+
+ if (!DstTy.isVector() || DstTy.getSizeInBits() > 128)
+ return false;
+
+ bool NeedExt = DstTy.getSizeInBits() > XTy.getSizeInBits();
+
+ MatchInfo = [=](MachineIRBuilder &B) {
+ CombinerHelper::applyAVG(B, Dst, X, Y, XTy, DstTy, IsSigned, NeedExt,
+ false);
+ };
+
+ return true;
+}
+
+// Fold shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
+bool CombinerHelper::matchAVGCeil(MachineInstr &MI, Register Ext1,
+ Register Ext2, Register ShiftAmtReg,
+ Register OneAmtReg, bool IsSigned,
+ MachineRegisterInfo &MRI,
+ BuildFnTy &MatchInfo) const {
+ assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
+ MI.getOpcode() == TargetOpcode::G_ASHR) &&
+ "Expected G_LSHR/G_ASHR");
+
+ if (!isOneOrOneSplat(ShiftAmtReg, false) ||
+ !isOneOrOneSplat(OneAmtReg, false))
+ return false;
+
+ Register X, Y;
+ if (IsSigned) {
+ if (!matchAnySExt(MRI, Ext1, X) || !matchAnySExt(MRI, Ext2, Y)) {
+ return false;
+ }
+ } else {
+ if (!mi_match(Ext1, MRI, m_GZExt(m_Reg(X))) ||
+ !mi_match(Ext2, MRI, m_GZExt(m_Reg(Y))))
+ return false;
+ }
+
+ LLT XTy = MRI.getType(X);
+ LLT YTy = MRI.getType(Y);
+
+ if (XTy != YTy)
+ return false;
+
+ Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
+
+ if (!DstTy.isVector() || DstTy.getSizeInBits() > 128)
+ return false;
+
+ bool NeedExt = DstTy.getSizeInBits() > XTy.getSizeInBits();
+
+ MatchInfo = [=](MachineIRBuilder &B) {
+ CombinerHelper::applyAVG(B, Dst, X, Y, XTy, DstTy, IsSigned, NeedExt, true);
+ };
+
+ return true;
+}
+
+void CombinerHelper::applyAVG(MachineIRBuilder &B, Register Dst, Register X,
+ Register Y, LLT XTy, LLT DstTy, bool IsSigned,
+ bool NeedExt, bool IsCeil) const {
+ unsigned TargetOpc;
+ if (IsCeil)
+ TargetOpc = IsSigned ? TargetOpcode::G_SAVGCEIL : TargetOpcode::G_UAVGCEIL;
+ else
+ TargetOpc =
+ IsSigned ? TargetOpcode::G_SAVGFLOOR : TargetOpcode::G_UAVGFLOOR;
+
+ if (!NeedExt) {
+ B.buildInstr(TargetOpc, {Dst}, {X, Y});
+ return;
+ }
+
+ if (DstTy.isVector() && XTy.getSizeInBits() < 64) {
+ Register ExtX = B.buildAnyExt(DstTy, X).getReg(0);
+ Register ExtY = B.buildAnyExt(DstTy, Y).getReg(0);
+ unsigned NarrowSize = XTy.getScalarSizeInBits();
+
+ Register PromotedX, PromotedY;
+ if (IsSigned) {
+ PromotedX = B.buildSExtInReg(DstTy, ExtX, NarrowSize).getReg(0);
+ PromotedY = B.buildSExtInReg(DstTy, ExtY, NarrowSize).getReg(0);
+ } else {
+ auto Mask = B.buildConstant(
+ DstTy, APInt::getLowBitsSet(DstTy.getScalarSizeInBits(), NarrowSize));
+ PromotedX = B.buildAnd(DstTy, ExtX, Mask).getReg(0);
+ PromotedY = B.buildAnd(DstTy, ExtY, Mask).getReg(0);
+ }
+
+ B.buildInstr(TargetOpc, {Dst}, {PromotedX, PromotedY});
+ return;
+ }
+
+ auto Avg = B.buildInstr(TargetOpc, {XTy}, {X, Y});
+ if (IsSigned)
+ B.buildSExt(Dst, Avg);
+ else
+ B.buildZExt(Dst, Avg);
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
new file mode 100644
index 0000000000000..afeee92150c4d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-avg.mir
@@ -0,0 +1,358 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner %s -o - | FileCheck %s
+
+...
+---
+name: test_uavgfloor_i16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_uavgfloor_i16
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+ ; CHECK-NEXT: $w0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 1
+ %6:_(s32) = G_LSHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgfloor_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_uavgfloor_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: %ext0:_(<8 x s32>) = G_ZEXT [[COPY]](<8 x s16>)
+ ; CHECK-NEXT: %ext1:_(<8 x s32>) = G_ZEXT [[COPY1]](<8 x s16>)
+ ; CHECK-NEXT: %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+ ; CHECK-NEXT: %shift_cst:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32)
+ ; CHECK-NEXT: %shift:_(<8 x s32>) = G_LSHR %sum, %v_shift_cst(<8 x s32>)
+ ; CHECK-NEXT: %res:_(<8 x s16>) = G_TRUNC %shift(<8 x s32>)
+ ; CHECK-NEXT: $q0 = COPY %res(<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+
+ %ext0:_(<8 x s32>) = G_ZEXT %0
+ %ext1:_(<8 x s32>) = G_ZEXT %1
+
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_LSHR %sum, %v_shift_cst
+
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_savgfloor_i16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_savgfloor_i16
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG %w0_copy, 16
+ ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG %w1_copy, 16
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG]], [[SEXT_INREG1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
+ ; CHECK-NEXT: $w0 = COPY [[ASHR]](s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_SEXT %0(s16)
+ %3:_(s32) = G_SEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 1
+ %6:_(s32) = G_ASHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_savgfloor_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_savgfloor_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: %ext0:_(<8 x s32>) = G_SEXT [[COPY]](<8 x s16>)
+ ; CHECK-NEXT: %ext1:_(<8 x s32>) = G_SEXT [[COPY1]](<8 x s16>)
+ ; CHECK-NEXT: %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+ ; CHECK-NEXT: %shift_cst:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32)
+ ; CHECK-NEXT: %shift:_(<8 x s32>) = G_ASHR %sum, %v_shift_cst(<8 x s32>)
+ ; CHECK-NEXT: %res:_(<8 x s16>) = G_TRUNC %shift(<8 x s32>)
+ ; CHECK-NEXT: $q0 = COPY %res(<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+
+ %ext0:_(<8 x s32>) = G_SEXT %0
+ %ext1:_(<8 x s32>) = G_SEXT %1
+
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_ASHR %sum, %v_shift_cst
+
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_negative_floor
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_negative_floor
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+ ; CHECK-NEXT: $w0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %4:_(s32) = G_ADD %2, %3
+
+ %5:_(s32) = G_CONSTANT i32 2
+ %6:_(s32) = G_LSHR %4, %5(s32)
+
+ %7:_(s16) = G_TRUNC %6(s32)
+
+ %ret_ext:_(s32) = G_ANYEXT %7(s16)
+ $w0 = COPY %ret_ext(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgceil_s32
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_uavgceil_s32
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: %sum:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: %cst1:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %add_one:_(s32) = G_ADD %sum, %cst1
+ ; CHECK-NEXT: %cst_shift:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %res:_(s32) = G_LSHR %add_one, %cst_shift(s32)
+ ; CHECK-NEXT: $w0 = COPY %res(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+
+ %sum:_(s32) = G_ADD %2, %3
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %add_one:_(s32) = G_ADD %sum, %cst1
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_LSHR %add_one, %cst_shift
+
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_savgceil_s32_associative
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_savgceil_s32_associative
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG %w0_copy, 16
+ ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG %w1_copy, 16
+ ; CHECK-NEXT: %cst1:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG]], [[SEXT_INREG1]]
+ ; CHECK-NEXT: %sum:_(s32) = G_ADD [[ADD]], %cst1
+ ; CHECK-NEXT: %cst_shift:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %res:_(s32) = G_ASHR %sum, %cst_shift(s32)
+ ; CHECK-NEXT: $w0 = COPY %res(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_SEXT %0(s16)
+ %3:_(s32) = G_SEXT %1(s16)
+
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %add_one:_(s32) = G_ADD %2, %cst1
+ %sum:_(s32) = G_ADD %add_one, %3
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_ASHR %sum, %cst_shift
+
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
+
+...
+---
+name: test_uavgceil_v8s16
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $q0, $q1
+ ; CHECK-LABEL: name: test_uavgceil_v8s16
+ ; CHECK: liveins: $q0, $q1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+ ; CHECK-NEXT: %ext0:_(<8 x s32>) = G_ZEXT [[COPY]](<8 x s16>)
+ ; CHECK-NEXT: %ext1:_(<8 x s32>) = G_ZEXT [[COPY1]](<8 x s16>)
+ ; CHECK-NEXT: %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+ ; CHECK-NEXT: %cst1:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %v_cst1:_(<8 x s32>) = G_BUILD_VECTOR %cst1(s32), %cst1(s32), %cst1(s32), %cst1(s32), %cst1(s32), %cst1(s32), %cst1(s32), %cst1(s32)
+ ; CHECK-NEXT: %add_one:_(<8 x s32>) = G_ADD %sum, %v_cst1
+ ; CHECK-NEXT: %shift_cst:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32), %shift_cst(s32)
+ ; CHECK-NEXT: %shift:_(<8 x s32>) = G_LSHR %add_one, %v_shift_cst(<8 x s32>)
+ ; CHECK-NEXT: %res:_(<8 x s16>) = G_TRUNC %shift(<8 x s32>)
+ ; CHECK-NEXT: $q0 = COPY %res(<8 x s16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %0:_(<8 x s16>) = COPY $q0
+ %1:_(<8 x s16>) = COPY $q1
+ %ext0:_(<8 x s32>) = G_ZEXT %0
+ %ext1:_(<8 x s32>) = G_ZEXT %1
+ %sum:_(<8 x s32>) = G_ADD %ext0, %ext1
+ %cst1:_(s32) = G_CONSTANT i32 1
+ %v_cst1:_(<8 x s32>) = G_BUILD_VECTOR %cst1, %cst1, %cst1, %cst1, %cst1, %cst1, %cst1, %cst1
+ %add_one:_(<8 x s32>) = G_ADD %sum, %v_cst1
+ %shift_cst:_(s32) = G_CONSTANT i32 1
+ %v_shift_cst:_(<8 x s32>) = G_BUILD_VECTOR %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst, %shift_cst
+ %shift:_(<8 x s32>) = G_LSHR %add_one, %v_shift_cst
+ %res:_(<8 x s16>) = G_TRUNC %shift
+ $q0 = COPY %res
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: test_negative_ceil_bad_constant
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: test_negative_ceil_bad_constant
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w0_copy:_(s32) = COPY $w0
+ ; CHECK-NEXT: %w1_copy:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC %w0_copy(s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC %w1_copy(s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: %sum:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: %cst2:_(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: %add_two:_(s32) = G_ADD %sum, %cst2
+ ; CHECK-NEXT: %cst_shift:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %res:_(s32) = G_LSHR %add_two, %cst_shift(s32)
+ ; CHECK-NEXT: $w0 = COPY %res(s32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w0_copy:_(s32) = COPY $w0
+ %w1_copy:_(s32) = COPY $w1
+ %0:_(s16) = G_TRUNC %w0_copy(s32)
+ %1:_(s16) = G_TRUNC %w1_copy(s32)
+ %2:_(s32) = G_ZEXT %0(s16)
+ %3:_(s32) = G_ZEXT %1(s16)
+ %sum:_(s32) = G_ADD %2, %3
+ %cst2:_(s32) = G_CONSTANT i32 2
+ %add_two:_(s32) = G_ADD %sum, %cst2
+ %cst_shift:_(s32) = G_CONSTANT i32 1
+ %res:_(s32) = G_LSHR %add_two, %cst_shift
+ $w0 = COPY %res(s32)
+ RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
index a1a6a08c5051a..d41bef7be3ecf 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
@@ -631,20 +631,11 @@ define void @testLowerToSHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture
}
define void @testLowerToURHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD8b:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.8b v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD8b:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.8h v2, #1
-; CHECK-GI-NEXT: uaddl.8h v0, v0, v1
-; CHECK-GI-NEXT: add.8h v0, v0, v2
-; CHECK-GI-NEXT: shrn.8b v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD8b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.8b v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i8> %src1 to <8 x i16>
%zextsrc2 = zext <8 x i8> %src2 to <8 x i16>
%add1 = add nuw nsw <8 x i16> %zextsrc1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
@@ -656,20 +647,11 @@ define void @testLowerToURHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture w
}
define void @testLowerToURHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD4h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD4h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi.4s v2, #1
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: add.4s v0, v0, v2
-; CHECK-GI-NEXT: shrn.4h v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD4h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.4h v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
%add1 = add nuw nsw <4 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1>
@@ -681,21 +663,11 @@ define void @testLowerToURHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture
}
define void @testLowerToURHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToURHADD2s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToURHADD2s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI38_0
-; CHECK-GI-NEXT: uaddl.2d v0, v0, v1
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI38_0]
-; CHECK-GI-NEXT: add.2d v0, v0, v1
-; CHECK-GI-NEXT: shrn.2s v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToURHADD2s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i32> %src1 to <2 x i64>
%zextsrc2 = zext <2 x i32> %src2 to <2 x i64>
%add1 = add nuw nsw <2 x i64> %zextsrc1, <i64 1, i64 1>
@@ -792,18 +764,11 @@ define void @testLowerToURHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture
}
define void @testLowerToUHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD8b:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.8b v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD8b:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.8h v0, v0, v1
-; CHECK-GI-NEXT: shrn.8b v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD8b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.8b v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i8> %src1 to <8 x i16>
%zextsrc2 = zext <8 x i8> %src2 to <8 x i16>
%add = add nuw nsw <8 x i16> %zextsrc1, %zextsrc2
@@ -814,18 +779,11 @@ define void @testLowerToUHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture wr
}
define void @testLowerToUHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD4h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD4h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: shrn.4h v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD4h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4h v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
%add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2
@@ -836,18 +794,11 @@ define void @testLowerToUHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture
}
define void @testLowerToUHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {
-; CHECK-SD-LABEL: testLowerToUHADD2s:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: str d0, [x0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testLowerToUHADD2s:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.2d v0, v0, v1
-; CHECK-GI-NEXT: shrn.2s v0, v0, #1
-; CHECK-GI-NEXT: str d0, [x0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testLowerToUHADD2s:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i32> %src1 to <2 x i64>
%zextsrc2 = zext <2 x i32> %src2 to <2 x i64>
%add = add nuw nsw <2 x i64> %zextsrc1, %zextsrc2
@@ -930,17 +881,11 @@ define void @testLowerToUHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture
}
define <4 x i32> @hadd16_sext_asr(<4 x i16> %src1, <4 x i16> %src2) {
-; CHECK-SD-LABEL: hadd16_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shadd.4h v0, v0, v1
-; CHECK-SD-NEXT: sshll.4s v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd16_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: saddl.4s v0, v0, v1
-; CHECK-GI-NEXT: sshr.4s v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd16_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shadd.4h v0, v0, v1
+; CHECK-NEXT: sshll.4s v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = sext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = sext <4 x i16> %src2 to <4 x i32>
%add = add nsw <4 x i32> %zextsrc1, %zextsrc2
@@ -949,17 +894,11 @@ define <4 x i32> @hadd16_sext_asr(<4 x i16> %src1, <4 x i16> %src2) {
}
define <4 x i32> @hadd16_zext_asr(<4 x i16> %src1, <4 x i16> %src2) {
-; CHECK-SD-LABEL: hadd16_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: ushll.4s v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd16_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: ushr.4s v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd16_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4h v0, v0, v1
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
%add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2
@@ -981,17 +920,11 @@ define <4 x i32> @hadd16_sext_lsr(<4 x i16> %src1, <4 x i16> %src2) {
}
define <4 x i32> @hadd16_zext_lsr(<4 x i16> %src1, <4 x i16> %src2) {
-; CHECK-SD-LABEL: hadd16_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uhadd.4h v0, v0, v1
-; CHECK-SD-NEXT: ushll.4s v0, v0, #0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd16_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uaddl.4s v0, v0, v1
-; CHECK-GI-NEXT: ushr.4s v0, v0, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd16_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uhadd.4h v0, v0, v1
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: ret
%zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
%zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
%add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2
@@ -1100,11 +1033,7 @@ define <4 x i16> @hadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
;
; CHECK-GI-LABEL: hadd8_sext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.4h v1, v1, #8
-; CHECK-GI-NEXT: shl.4h v0, v0, #8
-; CHECK-GI-NEXT: sshr.4h v1, v1, #8
-; CHECK-GI-NEXT: ssra.4h v1, v0, #8
-; CHECK-GI-NEXT: sshr.4h v0, v1, #1
+; CHECK-GI-NEXT: shadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = sext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = sext <4 x i8> %src2 to <4 x i16>
@@ -1126,8 +1055,7 @@ define <4 x i16> @hadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {
; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT: and.8b v0, v0, v2
; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
+; CHECK-GI-NEXT: uhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -1174,8 +1102,7 @@ define <4 x i16> @hadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT: and.8b v0, v0, v2
; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
+; CHECK-GI-NEXT: uhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -1185,30 +1112,14 @@ define <4 x i16> @hadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shl.2s v1, v1, #24
-; CHECK-SD-NEXT: shl.2s v0, v0, #24
-; CHECK-SD-NEXT: sshr.2s v1, v1, #24
-; CHECK-SD-NEXT: sshr.2s v0, v0, #24
-; CHECK-SD-NEXT: shadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.2s v1, v1, #24
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: shl.2s v0, v0, #24
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: sshr.2s v1, v1, #24
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: ssra.2s v1, v0, #24
-; CHECK-GI-NEXT: uzp1.4h v0, v1, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: sshl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl.2s v1, v1, #24
+; CHECK-NEXT: shl.2s v0, v0, #24
+; CHECK-NEXT: sshr.2s v1, v1, #24
+; CHECK-NEXT: sshr.2s v0, v0, #24
+; CHECK-NEXT: shadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
%add = add nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1217,29 +1128,13 @@ define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @hadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1282,29 +1177,13 @@ define <2 x i16> @hadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @hadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: hadd8x2_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: uhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: hadd8x2_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: mov.h v2[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v2
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: hadd8x2_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: uhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1324,13 +1203,7 @@ define <4 x i16> @rhadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
;
; CHECK-GI-LABEL: rhadd8_sext_asr:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.4h v1, v1, #8
-; CHECK-GI-NEXT: shl.4h v0, v0, #8
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: sshr.4h v1, v1, #8
-; CHECK-GI-NEXT: ssra.4h v1, v0, #8
-; CHECK-GI-NEXT: add.4h v0, v1, v2
-; CHECK-GI-NEXT: sshr.4h v0, v0, #1
+; CHECK-GI-NEXT: srhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = sext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = sext <4 x i8> %src2 to <4 x i16>
@@ -1353,10 +1226,7 @@ define <4 x i16> @rhadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {
; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT: and.8b v0, v0, v2
; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: add.4h v0, v0, v2
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
+; CHECK-GI-NEXT: urhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -1409,10 +1279,7 @@ define <4 x i16> @rhadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT: and.8b v0, v0, v2
; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.4h v2, #1
-; CHECK-GI-NEXT: add.4h v0, v0, v1
-; CHECK-GI-NEXT: add.4h v0, v0, v2
-; CHECK-GI-NEXT: ushr.4h v0, v0, #1
+; CHECK-GI-NEXT: urhadd.4h v0, v0, v1
; CHECK-GI-NEXT: ret
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -1423,32 +1290,14 @@ define <4 x i16> @rhadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_sext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shl.2s v1, v1, #24
-; CHECK-SD-NEXT: shl.2s v0, v0, #24
-; CHECK-SD-NEXT: sshr.2s v1, v1, #24
-; CHECK-SD-NEXT: sshr.2s v0, v0, #24
-; CHECK-SD-NEXT: srhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_sext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: shl.2s v1, v1, #24
-; CHECK-GI-NEXT: shl.2s v0, v0, #24
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: sshr.2s v1, v1, #24
-; CHECK-GI-NEXT: ssra.2s v1, v0, #24
-; CHECK-GI-NEXT: fmov s0, w8
-; CHECK-GI-NEXT: mov.h v0[1], w8
-; CHECK-GI-NEXT: add.2s v1, v1, v2
-; CHECK-GI-NEXT: uzp1.4h v1, v1, v0
-; CHECK-GI-NEXT: neg.4h v0, v0
-; CHECK-GI-NEXT: sshl.4h v0, v1, v0
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_sext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl.2s v1, v1, #24
+; CHECK-NEXT: shl.2s v0, v0, #24
+; CHECK-NEXT: sshr.2s v1, v1, #24
+; CHECK-NEXT: sshr.2s v0, v0, #24
+; CHECK-NEXT: srhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
%add = add nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1458,31 +1307,13 @@ define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_zext_asr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_zext_asr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: fmov s1, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v2
-; CHECK-GI-NEXT: mov.h v1[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v1
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_zext_asr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
@@ -1531,31 +1362,13 @@ define <2 x i16> @rhadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
}
define <2 x i16> @rhadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
-; CHECK-SD-LABEL: rhadd8x2_zext_lsr:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-SD-NEXT: and.8b v1, v1, v2
-; CHECK-SD-NEXT: and.8b v0, v0, v2
-; CHECK-SD-NEXT: urhadd.2s v0, v0, v1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadd8x2_zext_lsr:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff
-; CHECK-GI-NEXT: mov w8, #1 // =0x1
-; CHECK-GI-NEXT: and.8b v0, v0, v2
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: movi.2s v2, #1
-; CHECK-GI-NEXT: add.2s v0, v0, v1
-; CHECK-GI-NEXT: fmov s1, w8
-; CHECK-GI-NEXT: add.2s v0, v0, v2
-; CHECK-GI-NEXT: mov.h v1[1], w8
-; CHECK-GI-NEXT: uzp1.4h v0, v0, v0
-; CHECK-GI-NEXT: neg.4h v1, v1
-; CHECK-GI-NEXT: ushl.4h v0, v0, v1
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadd8x2_zext_lsr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and.8b v1, v1, v2
+; CHECK-NEXT: and.8b v0, v0, v2
+; CHECK-NEXT: urhadd.2s v0, v0, v1
+; CHECK-NEXT: ret
%zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
%zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
%add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
More information about the llvm-commits
mailing list