[llvm] [AArch64] SVE Shuffleopt: merge reduction reverse into tbl (PR #206047)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 05:26:27 PDT 2026
https://github.com/huntergr-arm created https://github.com/llvm/llvm-project/pull/206047
Building on top of the work to convert deinterleave + extend to tbl, this PR will search for a reverse on the other side of binops that use the extended value, which are themselves used in an fadd reduction. We can then fold the reverse into the tbls and remove the reverse from the loop.
Will work best when the cost model work to help select lower VFs for deinterleaving (#205844) lands.
>From fdb115e98781bbd5a4dcfc052cec6386537aea85 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 25 Jun 2026 14:52:48 +0000
Subject: [PATCH 1/2] Add tests with a reverse on the other operand of the
extend user
---
.../CodeGen/AArch64/sve-tbl-folding-opts.ll | 440 ++++++++++++++++++
1 file changed, 440 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll b/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
index e101489c564c8..fa3df9f39a4ad 100644
--- a/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
@@ -638,5 +638,445 @@ exit:
ret void
}
+define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double(ptr %src, ptr %src2, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z7.d, #0, #4
+; CHECK-NEXT: mov z2.d, #0xffffffffffff0000
+; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
+; CHECK-NEXT: mov x9, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z6.d, x9
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: add x9, x1, #8
+; CHECK-NEXT: add z4.d, z7.d, z2.d
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: cntw x10
+; CHECK-NEXT: add z5.d, z7.d, z5.d
+; CHECK-NEXT: add z6.d, z7.d, z6.d
+; CHECK-NEXT: rdvl x11, #2
+; CHECK-NEXT: add z7.d, z7.d, z24.d
+; CHECK-NEXT: .LBB8_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1h { z24.h }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z28.d }, p1/z, [x9, x8, lsl #3]
+; CHECK-NEXT: sub x8, x8, x10
+; CHECK-NEXT: cmn x8, #2048
+; CHECK-NEXT: add x0, x0, x11
+; CHECK-NEXT: tbl z25.h, { z24.h }, z4.h
+; CHECK-NEXT: tbl z26.h, { z24.h }, z5.h
+; CHECK-NEXT: tbl z27.h, { z24.h }, z6.h
+; CHECK-NEXT: tbl z24.h, { z24.h }, z7.h
+; CHECK-NEXT: rev z28.d, z28.d
+; CHECK-NEXT: ucvtf z25.d, p1/m, z25.d
+; CHECK-NEXT: ucvtf z26.d, p1/m, z26.d
+; CHECK-NEXT: ucvtf z27.d, p1/m, z27.d
+; CHECK-NEXT: ucvtf z24.d, p1/m, z24.d
+; CHECK-NEXT: fmul z25.d, z25.d, z28.d
+; CHECK-NEXT: fmul z26.d, z26.d, z28.d
+; CHECK-NEXT: fmul z27.d, z27.d, z28.d
+; CHECK-NEXT: fmul z24.d, z24.d, z28.d
+; CHECK-NEXT: fadd z0.d, z0.d, z25.d
+; CHECK-NEXT: fadd z3.d, z3.d, z26.d
+; CHECK-NEXT: fadd z1.d, z1.d, z27.d
+; CHECK-NEXT: fadd z2.d, z2.d, z24.d
+; CHECK-NEXT: b.ne .LBB8_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: faddv d0, p1, z0.d
+; CHECK-NEXT: faddv d3, p1, z3.d
+; CHECK-NEXT: faddv d1, p1, z1.d
+; CHECK-NEXT: faddv d2, p1, z2.d
+; CHECK-NEXT: mov v0.d[1], v3.d[0]
+; CHECK-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-NEXT: stp q0, q1, [x2]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 2
+ %common.base = getelementptr double, ptr %src2, i64 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.b.f64, %loop ]
+ %acc.g.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.g.f64, %loop ]
+ %acc.r.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.r.f64, %loop ]
+ %acc.a.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.a.f64, %loop ]
+ %negated = mul i64 %iv, -1
+ %common.term.ptr = getelementptr inbounds nuw double, ptr %common.base, i64 %negated
+ %common.term = load <vscale x 2 x double>, ptr %common.term.ptr, align 8
+ %reversed = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %common.term)
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.f64 = uitofp <vscale x 2 x i16> %b.i16 to <vscale x 2 x double>
+ %g.f64 = uitofp <vscale x 2 x i16> %g.i16 to <vscale x 2 x double>
+ %r.f64 = uitofp <vscale x 2 x i16> %r.i16 to <vscale x 2 x double>
+ %a.f64 = uitofp <vscale x 2 x i16> %a.i16 to <vscale x 2 x double>
+ %b.mul.f64 = fmul <vscale x 2 x double> %b.f64, %reversed
+ %g.mul.f64 = fmul <vscale x 2 x double> %g.f64, %reversed
+ %r.mul.f64 = fmul <vscale x 2 x double> %r.f64, %reversed
+ %a.mul.f64 = fmul <vscale x 2 x double> %a.f64, %reversed
+ %fadd.b.f64 = fadd <vscale x 2 x double> %acc.b.f64, %b.mul.f64
+ %fadd.g.f64 = fadd <vscale x 2 x double> %acc.g.f64, %g.mul.f64
+ %fadd.r.f64 = fadd <vscale x 2 x double> %acc.r.f64, %r.mul.f64
+ %fadd.a.f64 = fadd <vscale x 2 x double> %acc.a.f64, %a.mul.f64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %b.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.b.f64)
+ store double %b.acc, ptr %dst
+ %g.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.g.f64)
+ %g.f64.gep = getelementptr double, ptr %dst, i64 1
+ store double %g.acc, ptr %g.f64.gep
+ %r.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.r.f64)
+ %r.f64.gep = getelementptr double, ptr %dst, i64 2
+ store double %r.acc, ptr %r.f64.gep
+ %a.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.a.f64)
+ %a.f64.gep = getelementptr double, ptr %dst, i64 3
+ store double %a.acc, ptr %a.f64.gep
+ ret void
+}
+
+define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_partial_coverage(ptr %src, ptr %src2, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_partial_coverage:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z7.d, #0, #4
+; CHECK-NEXT: mov z2.d, #0xffffffffffff0000
+; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
+; CHECK-NEXT: mov x9, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z6.d, x9
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: add x9, x1, #8
+; CHECK-NEXT: add z4.d, z7.d, z2.d
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: cntw x10
+; CHECK-NEXT: add z5.d, z7.d, z5.d
+; CHECK-NEXT: add z6.d, z7.d, z6.d
+; CHECK-NEXT: rdvl x11, #2
+; CHECK-NEXT: add z7.d, z7.d, z24.d
+; CHECK-NEXT: .LBB9_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1h { z24.h }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z28.d }, p1/z, [x9, x8, lsl #3]
+; CHECK-NEXT: sub x8, x8, x10
+; CHECK-NEXT: cmn x8, #2048
+; CHECK-NEXT: add x0, x0, x11
+; CHECK-NEXT: tbl z25.h, { z24.h }, z4.h
+; CHECK-NEXT: tbl z26.h, { z24.h }, z5.h
+; CHECK-NEXT: tbl z27.h, { z24.h }, z6.h
+; CHECK-NEXT: tbl z24.h, { z24.h }, z7.h
+; CHECK-NEXT: rev z29.d, z28.d
+; CHECK-NEXT: ucvtf z25.d, p1/m, z25.d
+; CHECK-NEXT: ucvtf z26.d, p1/m, z26.d
+; CHECK-NEXT: ucvtf z27.d, p1/m, z27.d
+; CHECK-NEXT: ucvtf z24.d, p1/m, z24.d
+; CHECK-NEXT: fmul z25.d, z25.d, z29.d
+; CHECK-NEXT: fmul z26.d, z26.d, z28.d
+; CHECK-NEXT: fmul z27.d, z27.d, z29.d
+; CHECK-NEXT: fmul z24.d, z24.d, z28.d
+; CHECK-NEXT: fadd z0.d, z0.d, z25.d
+; CHECK-NEXT: fadd z3.d, z3.d, z26.d
+; CHECK-NEXT: fadd z1.d, z1.d, z27.d
+; CHECK-NEXT: fadd z2.d, z2.d, z24.d
+; CHECK-NEXT: b.ne .LBB9_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: faddv d0, p1, z0.d
+; CHECK-NEXT: faddv d3, p1, z3.d
+; CHECK-NEXT: faddv d1, p1, z1.d
+; CHECK-NEXT: faddv d2, p1, z2.d
+; CHECK-NEXT: mov v0.d[1], v3.d[0]
+; CHECK-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-NEXT: stp q0, q1, [x2]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 2
+ %common.base = getelementptr double, ptr %src2, i64 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.b.f64, %loop ]
+ %acc.g.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.g.f64, %loop ]
+ %acc.r.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.r.f64, %loop ]
+ %acc.a.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.a.f64, %loop ]
+ %negated = mul i64 %iv, -1
+ %common.term.ptr = getelementptr inbounds nuw double, ptr %common.base, i64 %negated
+ %common.term = load <vscale x 2 x double>, ptr %common.term.ptr, align 8
+ %reversed = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %common.term)
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.f64 = uitofp <vscale x 2 x i16> %b.i16 to <vscale x 2 x double>
+ %g.f64 = uitofp <vscale x 2 x i16> %g.i16 to <vscale x 2 x double>
+ %r.f64 = uitofp <vscale x 2 x i16> %r.i16 to <vscale x 2 x double>
+ %a.f64 = uitofp <vscale x 2 x i16> %a.i16 to <vscale x 2 x double>
+ %b.mul.f64 = fmul <vscale x 2 x double> %b.f64, %reversed
+ %g.mul.f64 = fmul <vscale x 2 x double> %g.f64, %common.term
+ %r.mul.f64 = fmul <vscale x 2 x double> %r.f64, %reversed
+ %a.mul.f64 = fmul <vscale x 2 x double> %a.f64, %common.term
+ %fadd.b.f64 = fadd <vscale x 2 x double> %acc.b.f64, %b.mul.f64
+ %fadd.g.f64 = fadd <vscale x 2 x double> %acc.g.f64, %g.mul.f64
+ %fadd.r.f64 = fadd <vscale x 2 x double> %acc.r.f64, %r.mul.f64
+ %fadd.a.f64 = fadd <vscale x 2 x double> %acc.a.f64, %a.mul.f64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %b.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.b.f64)
+ store double %b.acc, ptr %dst
+ %g.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.g.f64)
+ %g.f64.gep = getelementptr double, ptr %dst, i64 1
+ store double %g.acc, ptr %g.f64.gep
+ %r.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.r.f64)
+ %r.f64.gep = getelementptr double, ptr %dst, i64 2
+ store double %r.acc, ptr %r.f64.gep
+ %a.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.a.f64)
+ %a.f64.gep = getelementptr double, ptr %dst, i64 3
+ store double %a.acc, ptr %a.f64.gep
+ ret void
+}
+
+;; No reduction, so we cannot safely reorder the lanes.
+define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double_no_rdx(ptr %src, ptr %src2, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double_no_rdx:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z7.d, #0, #4
+; CHECK-NEXT: mov z3.d, #0xffffffffffff0000
+; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
+; CHECK-NEXT: mov x9, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z6.d, x9
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: add x9, x1, #8
+; CHECK-NEXT: add z4.d, z7.d, z3.d
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: cntw x10
+; CHECK-NEXT: add z5.d, z7.d, z5.d
+; CHECK-NEXT: add z6.d, z7.d, z6.d
+; CHECK-NEXT: rdvl x11, #2
+; CHECK-NEXT: add z7.d, z7.d, z24.d
+; CHECK-NEXT: .LBB10_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1h { z24.h }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z28.d }, p1/z, [x9, x8, lsl #3]
+; CHECK-NEXT: sub x8, x8, x10
+; CHECK-NEXT: cmn x8, #2048
+; CHECK-NEXT: add x0, x0, x11
+; CHECK-NEXT: tbl z25.h, { z24.h }, z4.h
+; CHECK-NEXT: tbl z26.h, { z24.h }, z5.h
+; CHECK-NEXT: tbl z27.h, { z24.h }, z6.h
+; CHECK-NEXT: tbl z24.h, { z24.h }, z7.h
+; CHECK-NEXT: rev z28.d, z28.d
+; CHECK-NEXT: ucvtf z25.d, p1/m, z25.d
+; CHECK-NEXT: ucvtf z26.d, p1/m, z26.d
+; CHECK-NEXT: ucvtf z27.d, p1/m, z27.d
+; CHECK-NEXT: ucvtf z24.d, p1/m, z24.d
+; CHECK-NEXT: fmul z25.d, z25.d, z28.d
+; CHECK-NEXT: fmul z26.d, z26.d, z28.d
+; CHECK-NEXT: fmul z27.d, z27.d, z28.d
+; CHECK-NEXT: fmul z24.d, z24.d, z28.d
+; CHECK-NEXT: fadd z0.d, z0.d, z25.d
+; CHECK-NEXT: fadd z1.d, z1.d, z26.d
+; CHECK-NEXT: fadd z2.d, z2.d, z27.d
+; CHECK-NEXT: fadd z3.d, z3.d, z24.d
+; CHECK-NEXT: b.ne .LBB10_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: str z0, [x2]
+; CHECK-NEXT: str z1, [x2, #1, mul vl]
+; CHECK-NEXT: str z2, [x2, #2, mul vl]
+; CHECK-NEXT: str z3, [x2, #3, mul vl]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 2
+ %common.base = getelementptr double, ptr %src2, i64 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.b.f64, %loop ]
+ %acc.g.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.g.f64, %loop ]
+ %acc.r.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.r.f64, %loop ]
+ %acc.a.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.a.f64, %loop ]
+ %negated = mul i64 %iv, -1
+ %common.term.ptr = getelementptr inbounds nuw double, ptr %common.base, i64 %negated
+ %common.term = load <vscale x 2 x double>, ptr %common.term.ptr, align 8
+ %reversed = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %common.term)
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.f64 = uitofp <vscale x 2 x i16> %b.i16 to <vscale x 2 x double>
+ %g.f64 = uitofp <vscale x 2 x i16> %g.i16 to <vscale x 2 x double>
+ %r.f64 = uitofp <vscale x 2 x i16> %r.i16 to <vscale x 2 x double>
+ %a.f64 = uitofp <vscale x 2 x i16> %a.i16 to <vscale x 2 x double>
+ %b.mul.f64 = fmul <vscale x 2 x double> %b.f64, %reversed
+ %g.mul.f64 = fmul <vscale x 2 x double> %g.f64, %reversed
+ %r.mul.f64 = fmul <vscale x 2 x double> %r.f64, %reversed
+ %a.mul.f64 = fmul <vscale x 2 x double> %a.f64, %reversed
+ %fadd.b.f64 = fadd <vscale x 2 x double> %acc.b.f64, %b.mul.f64
+ %fadd.g.f64 = fadd <vscale x 2 x double> %acc.g.f64, %g.mul.f64
+ %fadd.r.f64 = fadd <vscale x 2 x double> %acc.r.f64, %r.mul.f64
+ %fadd.a.f64 = fadd <vscale x 2 x double> %acc.a.f64, %a.mul.f64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store <vscale x 2 x double> %fadd.b.f64, ptr %dst
+ %g.f64.gep = getelementptr <vscale x 2 x double>, ptr %dst, i64 1
+ store <vscale x 2 x double> %fadd.g.f64, ptr %g.f64.gep
+ %r.f64.gep = getelementptr <vscale x 2 x double>, ptr %dst, i64 2
+ store <vscale x 2 x double> %fadd.r.f64, ptr %r.f64.gep
+ %a.f64.gep = getelementptr <vscale x 2 x double>, ptr %dst, i64 3
+ store <vscale x 2 x double> %fadd.a.f64, ptr %a.f64.gep
+ ret void
+}
+
+;; Skip folding the rev into the tbl because it has an extra use.
+define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double_extra_use(ptr %src, ptr %src2, ptr %dst, ptr %dst2, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double_extra_use:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z7.d, #0, #4
+; CHECK-NEXT: mov z1.d, #0xffffffffffff0000
+; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
+; CHECK-NEXT: mov x10, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z6.d, x10
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov x9, xzr
+; CHECK-NEXT: add z2.d, z7.d, z1.d
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: add x10, x1, #8
+; CHECK-NEXT: add z5.d, z7.d, z5.d
+; CHECK-NEXT: add z6.d, z7.d, z6.d
+; CHECK-NEXT: cntw x11
+; CHECK-NEXT: add z7.d, z7.d, z24.d
+; CHECK-NEXT: rdvl x12, #2
+; CHECK-NEXT: .LBB11_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1d { z24.d }, p1/z, [x10, x8, lsl #3]
+; CHECK-NEXT: sub x8, x8, x11
+; CHECK-NEXT: add x9, x9, x11
+; CHECK-NEXT: cmn x8, #2048
+; CHECK-NEXT: rev z24.d, z24.d
+; CHECK-NEXT: str z24, [x3]
+; CHECK-NEXT: ld1h { z25.h }, p0/z, [x0]
+; CHECK-NEXT: add x0, x0, x12
+; CHECK-NEXT: tbl z26.h, { z25.h }, z2.h
+; CHECK-NEXT: tbl z27.h, { z25.h }, z5.h
+; CHECK-NEXT: tbl z28.h, { z25.h }, z6.h
+; CHECK-NEXT: tbl z25.h, { z25.h }, z7.h
+; CHECK-NEXT: ucvtf z26.d, p1/m, z26.d
+; CHECK-NEXT: ucvtf z27.d, p1/m, z27.d
+; CHECK-NEXT: ucvtf z28.d, p1/m, z28.d
+; CHECK-NEXT: ucvtf z25.d, p1/m, z25.d
+; CHECK-NEXT: fmul z26.d, z26.d, z24.d
+; CHECK-NEXT: fmul z27.d, z27.d, z24.d
+; CHECK-NEXT: fmul z28.d, z28.d, z24.d
+; CHECK-NEXT: fmul z24.d, z25.d, z24.d
+; CHECK-NEXT: fadd z0.d, z0.d, z26.d
+; CHECK-NEXT: fadd z4.d, z4.d, z27.d
+; CHECK-NEXT: fadd z3.d, z3.d, z28.d
+; CHECK-NEXT: fadd z1.d, z1.d, z24.d
+; CHECK-NEXT: b.ne .LBB11_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: faddv d0, p1, z0.d
+; CHECK-NEXT: faddv d2, p1, z4.d
+; CHECK-NEXT: faddv d3, p1, z3.d
+; CHECK-NEXT: faddv d1, p1, z1.d
+; CHECK-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-NEXT: mov v3.d[1], v1.d[0]
+; CHECK-NEXT: stp q0, q3, [x2]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 2
+ %common.base = getelementptr double, ptr %src2, i64 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.b.f64, %loop ]
+ %acc.g.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.g.f64, %loop ]
+ %acc.r.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.r.f64, %loop ]
+ %acc.a.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.a.f64, %loop ]
+ %negated = mul i64 %iv, -1
+ %common.term.ptr = getelementptr inbounds nuw double, ptr %common.base, i64 %negated
+ %common.term = load <vscale x 2 x double>, ptr %common.term.ptr, align 8
+ %reversed = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %common.term)
+ %dst2.ptr = getelementptr inbounds nuw double, ptr %dst2, i64 %iv
+ store <vscale x 2 x double> %reversed, ptr %dst2, align 8
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.f64 = uitofp <vscale x 2 x i16> %b.i16 to <vscale x 2 x double>
+ %g.f64 = uitofp <vscale x 2 x i16> %g.i16 to <vscale x 2 x double>
+ %r.f64 = uitofp <vscale x 2 x i16> %r.i16 to <vscale x 2 x double>
+ %a.f64 = uitofp <vscale x 2 x i16> %a.i16 to <vscale x 2 x double>
+ %b.mul.f64 = fmul <vscale x 2 x double> %b.f64, %reversed
+ %g.mul.f64 = fmul <vscale x 2 x double> %g.f64, %reversed
+ %r.mul.f64 = fmul <vscale x 2 x double> %r.f64, %reversed
+ %a.mul.f64 = fmul <vscale x 2 x double> %a.f64, %reversed
+ %fadd.b.f64 = fadd <vscale x 2 x double> %acc.b.f64, %b.mul.f64
+ %fadd.g.f64 = fadd <vscale x 2 x double> %acc.g.f64, %g.mul.f64
+ %fadd.r.f64 = fadd <vscale x 2 x double> %acc.r.f64, %r.mul.f64
+ %fadd.a.f64 = fadd <vscale x 2 x double> %acc.a.f64, %a.mul.f64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %b.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.b.f64)
+ store double %b.acc, ptr %dst
+ %g.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.g.f64)
+ %g.f64.gep = getelementptr double, ptr %dst, i64 1
+ store double %g.acc, ptr %g.f64.gep
+ %r.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.r.f64)
+ %r.f64.gep = getelementptr double, ptr %dst, i64 2
+ store double %r.acc, ptr %r.f64.gep
+ %a.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %fadd.a.f64)
+ %a.f64.gep = getelementptr double, ptr %dst, i64 3
+ store double %a.acc, ptr %a.f64.gep
+ ret void
+}
+
attributes #0 = { "target-features"="+sve" }
attributes #1 = { "target-features"="+sve" vscale_range(1, 8) }
>From 6977cc32a47f0b643dc10200e2cf149cf058908d Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 25 Jun 2026 16:06:54 +0000
Subject: [PATCH 2/2] Fold reverses into the tbl where possible
---
llvm/lib/Target/AArch64/SVEShuffleOpts.cpp | 158 +++++++++++++++++-
.../CodeGen/AArch64/sve-tbl-folding-opts.ll | 81 +++++----
2 files changed, 196 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
index 61b32366153a9..d7f652a7a18f8 100644
--- a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
@@ -155,10 +155,130 @@ static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
Candidates.try_emplace(I, Extends);
}
+/// Evaluate a reverse intrinsic to see what uses it. We want to find a reverse
+/// that pairs with an extract from a deinterleave, so that we can move the
+/// reverse into the tbl as well as deinterleave and extend. We also need to
+/// confirm that it's only used by single-use instructions, or instructions
+/// used by a phi and a reduction intrinsic outside the loop, where the
+/// reduction permits reassociation. Something like the following:
+///
+/// %acc.b.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00),
+/// %entry ], [ %fadd.b.f64, %loop ]
+/// ...
+/// %rev.load = load <vscale x 2 x double>, ptr %rev.ptr
+/// %reversed = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(
+/// <vscale x 2 x double> %rev.load)
+/// %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep,
+/// <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+/// %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>,
+/// <vscale x 2 x i16>, <vscale x 2 x i16> }
+/// @llvm.vector.deinterleave4(<vscale x 8 x i16>
+/// %bgra)
+/// %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+/// <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+/// %b.f64 = uitofp <vscale x 2 x i16> %b.i16 to <vscale x 2 x double>
+/// %b.mul.f64 = fmul <vscale x 2 x double> %b.f64, %reversed
+/// %fadd.b.f64 = fadd <vscale x 2 x double> %acc.b.f64, %b.mul.f64
+/// %iv.next = add nuw i64 %iv, %stride
+/// %ec = icmp eq i64 %iv.next, 2048
+/// br i1 %ec, label %exit, label %loop
+/// ...
+/// %b.acc = call fast double @llvm.vector.reduce.fadd.nxv2f64(double
+/// 0.000000e+00, <vscale x 2 x double> %fadd.b.f64)
+static void evaluateReverse(IntrinsicInst *Reverse,
+ SmallVectorImpl<IntrinsicInst *> &Reverses, Loop &L,
+ const AArch64TargetLowering &TL,
+ const DataLayout DL) {
+ assert(Reverse->getIntrinsicID() == Intrinsic::vector_reverse &&
+ "Invalid intrinsic used");
+
+ // We want to check that for each use of the reverse, we eventually end up
+ // in a reassociable reduction, so that the ordering of elements does not
+ // matter.
+ for (User *U : Reverse->users()) {
+ Instruction *I = cast<Instruction>(U);
+
+ // For now, only iterate through normal operations until we hit a
+ // CallInst. If that's not a reduction, abandon.
+ // TODO: Handle more generic input when we have a use for it.
+ while (!isa<CallInst>(I)) {
+ // If there's only one use, proceeed to the next in the chain.
+ if (I->hasOneUse()) {
+ I = I->user_back();
+ continue;
+ }
+
+ // Otherwise, look for 2 users -- a phi in the loop, and an outside
+ // reduction intrinsic.
+ SmallVector<User *, 2> Users(I->users());
+ if (Users.size() != 2)
+ return;
+
+ Instruction *Phi = cast<Instruction>(Users[0]);
+ Instruction *Reduce = cast<Instruction>(Users[1]);
+ if (!isa<PHINode>(Phi))
+ std::swap(Phi, Reduce);
+
+ if (!isa<PHINode>(Phi) || !isa<IntrinsicInst>(Reduce))
+ return;
+
+ // TODO: Do we have a nice utility to match all reduction intrinsics?
+ // We need to confirm reassociation is allowed, because the lanes within
+ // the vector will be in reversed order.
+ IntrinsicInst *II = cast<IntrinsicInst>(Reduce);
+ if (!isa<FPMathOperator>(II) || !II->hasAllowReassoc() ||
+ II->getIntrinsicID() != Intrinsic::vector_reduce_fadd)
+ return;
+
+ // The Phi must be for the current loop, and the reduction must be outside
+ // it.
+ if (!L.contains(Phi) || L.contains(Reduce))
+ return;
+
+ I = Reduce;
+ }
+ }
+
+ Reverses.push_back(Reverse);
+}
+
/// Given a map of deinterleaves to zext or uitofp casts, remove the operations
/// and replace them with tbl shuffles.
-static void optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves) {
+static void
+optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves,
+ ArrayRef<IntrinsicInst *> Reverses) {
for (auto &[Deinterleave, Extends] : Deinterleaves) {
+
+ // See if there's a matching reverse.
+ // TODO: Support missing extends.
+ IntrinsicInst *Reverse = nullptr;
+ if (Extends[0]) {
+ User *ExtUser = Extends[0]->user_back();
+ for (auto *R : Reverses) {
+ if (is_contained(R->users(), ExtUser))
+ Reverse = R;
+ }
+ }
+
+ SmallVector<Instruction *, 4> ReversingExtends;
+ // Make sure all uses of the reverse can be matched to the deinterleave.
+ // Not all deinterleaved lanes need to be matched to a reverse, but all
+ // uses of the reverse must match a use of the deinterleave.
+ if (Reverse) {
+ for (auto *Ext : Extends) {
+ if (Ext && is_contained(Reverse->users(), Ext->user_back())) {
+ ReversingExtends.push_back(Ext);
+ }
+ }
+
+ // Bail out and just transform the deinterleave + extend if there are
+ // extra uses of the reverse not covered by the extracts/extends.
+ if (ReversingExtends.size() != range_size(Reverse->users())) {
+ ReversingExtends.clear();
+ Reverse = nullptr;
+ }
+ }
+
VectorType *DestTy = cast<VectorType>(Extends[0]->getDestTy());
VectorType *SrcTy = cast<VectorType>(Extends[0]->getSrcTy());
unsigned DstBits = DestTy->getScalarSizeInBits();
@@ -171,6 +291,7 @@ static void optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves) {
APInt Invalid = APInt::getAllOnes(DstBits);
for (auto [Idx, Extend] : enumerate(Extends)) {
// If not all lanes were extracted, we can have gaps. Skip over them.
+ bool ShouldReverse = is_contained(ReversingExtends, Extend);
if (!Extend)
continue;
// Build the mask using stepvectors and casting.
@@ -184,13 +305,27 @@ static void optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves) {
// step of 4. We then cast that back to an element size of 16b, yielding
// <0x0000 + Idx, 0xFFFF, 0xFFFF, 0xFFFF, 0x0004 + Idx, 0xFFFF...>.
APInt StartIdx = Invalid << SrcBits;
- StartIdx += Idx;
+ if (ShouldReverse) {
+ StartIdx = StartIdx - 4 + Idx;
+ } else {
+ StartIdx += Idx;
+ }
IRBuilder<> Builder(Extend);
Value *StepVector = Builder.CreateStepVector(StepVecTy);
+ uint64_t Step = ShouldReverse ? -4 : 4;
+ Value *Start = ConstantInt::get(StepVecTy, StartIdx);
+ if (ShouldReverse) {
+ Value *EltCnt = Builder.CreateVScale(Start->getType()->getScalarType());
+ EltCnt = Builder.CreateNUWMul(
+ EltCnt, ConstantInt::get(EltCnt->getType(), SrcBits / 8));
+ EltCnt =
+ Builder.CreateVectorSplat(StepVecTy->getElementCount(), EltCnt);
+ Start = Builder.CreateNUWAdd(Start, EltCnt);
+ }
+
Value *ScaledSteps =
- Builder.CreateNUWMul(StepVector, ConstantInt::get(StepVecTy, 4));
- Value *ZextTbl = Builder.CreateNUWAdd(
- ScaledSteps, ConstantInt::get(StepVecTy, StartIdx));
+ Builder.CreateNUWMul(StepVector, ConstantInt::get(StepVecTy, Step));
+ Value *ZextTbl = Builder.CreateNUWAdd(ScaledSteps, Start);
Value *FinalMask = Builder.CreateBitCast(ZextTbl, InputTy);
// Replace the deinterleave, extractvalue, and extension chain with
@@ -210,6 +345,11 @@ static void optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves) {
for (User *U : make_early_inc_range(Deinterleave->users()))
cast<Instruction>(U)->eraseFromParent();
Deinterleave->eraseFromParent();
+
+ if (Reverse) {
+ Reverse->replaceAllUsesWith(Reverse->getArgOperand(0));
+ Reverse->eraseFromParent();
+ }
}
}
@@ -227,15 +367,19 @@ static bool processLoop(Loop &L, const AArch64Subtarget &ST, DataLayout DL) {
assert(DL.isLittleEndian() &&
"Shuffle optimizations unsupported for big endian targets.");
DeinterleaveMap Candidates;
+ SmallVector<IntrinsicInst *, 2> Reverses;
for (auto *BB : L.blocks())
- for (auto &I : *BB)
+ for (auto &I : *BB) {
if (match(&I, m_Intrinsic<Intrinsic::vector_deinterleave4>(m_Value())))
evaluateDeinterleave(cast<IntrinsicInst>(&I), Candidates, L, TL, DL);
+ if (match(&I, m_Intrinsic<Intrinsic::vector_reverse>(m_Value())))
+ evaluateReverse(cast<IntrinsicInst>(&I), Reverses, L, TL, DL);
+ }
if (Candidates.empty())
return false;
- optimizeSVEDeinterleavedExtends(Candidates);
+ optimizeSVEDeinterleavedExtends(Candidates, Reverses);
return true;
}
diff --git a/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll b/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
index fa3df9f39a4ad..e9295f6bf5105 100644
--- a/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
@@ -641,24 +641,30 @@ exit:
define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double(ptr %src, ptr %src2, ptr %dst, <vscale x 8 x i1> %mask) #0 {
; CHECK-LABEL: uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: index z7.d, #0, #4
-; CHECK-NEXT: mov z2.d, #0xffffffffffff0000
-; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
-; CHECK-NEXT: mov x9, #-65534 // =0xffffffffffff0002
-; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
+; CHECK-NEXT: index z7.d, #0, #-4
+; CHECK-NEXT: mov x9, #-4 // =0xfffffffffffffffc
+; CHECK-NEXT: mov x10, #-3 // =0xfffffffffffffffd
+; CHECK-NEXT: movk x9, #65534, lsl #16
+; CHECK-NEXT: movk x10, #65534, lsl #16
+; CHECK-NEXT: mov z24.d, #0xfffffffffffeffff
+; CHECK-NEXT: mov z4.d, x9
+; CHECK-NEXT: mov x9, #-2 // =0xfffffffffffffffe
+; CHECK-NEXT: mov z5.d, x10
+; CHECK-NEXT: movk x9, #65534, lsl #16
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z6.d, x9
; CHECK-NEXT: movi v3.2d, #0000000000000000
-; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: incd z7.d
+; CHECK-NEXT: mov z6.d, x9
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: add x9, x1, #8
-; CHECK-NEXT: add z4.d, z7.d, z2.d
-; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: cntw x10
+; CHECK-NEXT: rdvl x11, #2
+; CHECK-NEXT: add z4.d, z7.d, z4.d
; CHECK-NEXT: add z5.d, z7.d, z5.d
; CHECK-NEXT: add z6.d, z7.d, z6.d
-; CHECK-NEXT: rdvl x11, #2
; CHECK-NEXT: add z7.d, z7.d, z24.d
; CHECK-NEXT: .LBB8_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
@@ -671,7 +677,6 @@ define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double(ptr
; CHECK-NEXT: tbl z26.h, { z24.h }, z5.h
; CHECK-NEXT: tbl z27.h, { z24.h }, z6.h
; CHECK-NEXT: tbl z24.h, { z24.h }, z7.h
-; CHECK-NEXT: rev z28.d, z28.d
; CHECK-NEXT: ucvtf z25.d, p1/m, z25.d
; CHECK-NEXT: ucvtf z26.d, p1/m, z26.d
; CHECK-NEXT: ucvtf z27.d, p1/m, z27.d
@@ -682,17 +687,17 @@ define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_double(ptr
; CHECK-NEXT: fmul z24.d, z24.d, z28.d
; CHECK-NEXT: fadd z0.d, z0.d, z25.d
; CHECK-NEXT: fadd z3.d, z3.d, z26.d
-; CHECK-NEXT: fadd z1.d, z1.d, z27.d
-; CHECK-NEXT: fadd z2.d, z2.d, z24.d
+; CHECK-NEXT: fadd z2.d, z2.d, z27.d
+; CHECK-NEXT: fadd z1.d, z1.d, z24.d
; CHECK-NEXT: b.ne .LBB8_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: faddv d0, p1, z0.d
; CHECK-NEXT: faddv d3, p1, z3.d
-; CHECK-NEXT: faddv d1, p1, z1.d
; CHECK-NEXT: faddv d2, p1, z2.d
+; CHECK-NEXT: faddv d1, p1, z1.d
; CHECK-NEXT: mov v0.d[1], v3.d[0]
-; CHECK-NEXT: mov v1.d[1], v2.d[0]
-; CHECK-NEXT: stp q0, q1, [x2]
+; CHECK-NEXT: mov v2.d[1], v1.d[0]
+; CHECK-NEXT: stp q0, q2, [x2]
; CHECK-NEXT: ret
entry:
%vscale = tail call i64 @llvm.vscale.i64()
@@ -751,25 +756,30 @@ exit:
define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_partial_coverage(ptr %src, ptr %src2, ptr %dst, <vscale x 8 x i1> %mask) #0 {
; CHECK-LABEL: uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_partial_coverage:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z5.d, #0, #-4
+; CHECK-NEXT: mov x9, #-4 // =0xfffffffffffffffc
+; CHECK-NEXT: mov x10, #-2 // =0xfffffffffffffffe
+; CHECK-NEXT: movk x9, #65534, lsl #16
; CHECK-NEXT: index z7.d, #0, #4
-; CHECK-NEXT: mov z2.d, #0xffffffffffff0000
-; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
-; CHECK-NEXT: mov x9, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z6.d, #0xffffffffffff0001
+; CHECK-NEXT: movk x10, #65534, lsl #16
; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
+; CHECK-NEXT: mov z4.d, x9
+; CHECK-NEXT: mov z25.d, x10
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z6.d, x9
; CHECK-NEXT: movi v3.2d, #0000000000000000
-; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: incd z5.d
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: add z6.d, z7.d, z6.d
; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: add z7.d, z7.d, z24.d
; CHECK-NEXT: add x9, x1, #8
-; CHECK-NEXT: add z4.d, z7.d, z2.d
-; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: cntw x10
-; CHECK-NEXT: add z5.d, z7.d, z5.d
-; CHECK-NEXT: add z6.d, z7.d, z6.d
+; CHECK-NEXT: add z4.d, z5.d, z4.d
+; CHECK-NEXT: add z5.d, z5.d, z25.d
; CHECK-NEXT: rdvl x11, #2
-; CHECK-NEXT: add z7.d, z7.d, z24.d
; CHECK-NEXT: .LBB9_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ld1h { z24.h }, p0/z, [x0]
@@ -778,31 +788,30 @@ define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_fma_with_reverse_partial_cov
; CHECK-NEXT: cmn x8, #2048
; CHECK-NEXT: add x0, x0, x11
; CHECK-NEXT: tbl z25.h, { z24.h }, z4.h
-; CHECK-NEXT: tbl z26.h, { z24.h }, z5.h
-; CHECK-NEXT: tbl z27.h, { z24.h }, z6.h
+; CHECK-NEXT: tbl z26.h, { z24.h }, z6.h
+; CHECK-NEXT: tbl z27.h, { z24.h }, z5.h
; CHECK-NEXT: tbl z24.h, { z24.h }, z7.h
-; CHECK-NEXT: rev z29.d, z28.d
; CHECK-NEXT: ucvtf z25.d, p1/m, z25.d
; CHECK-NEXT: ucvtf z26.d, p1/m, z26.d
; CHECK-NEXT: ucvtf z27.d, p1/m, z27.d
; CHECK-NEXT: ucvtf z24.d, p1/m, z24.d
-; CHECK-NEXT: fmul z25.d, z25.d, z29.d
+; CHECK-NEXT: fmul z25.d, z25.d, z28.d
; CHECK-NEXT: fmul z26.d, z26.d, z28.d
-; CHECK-NEXT: fmul z27.d, z27.d, z29.d
+; CHECK-NEXT: fmul z27.d, z27.d, z28.d
; CHECK-NEXT: fmul z24.d, z24.d, z28.d
; CHECK-NEXT: fadd z0.d, z0.d, z25.d
; CHECK-NEXT: fadd z3.d, z3.d, z26.d
-; CHECK-NEXT: fadd z1.d, z1.d, z27.d
-; CHECK-NEXT: fadd z2.d, z2.d, z24.d
+; CHECK-NEXT: fadd z2.d, z2.d, z27.d
+; CHECK-NEXT: fadd z1.d, z1.d, z24.d
; CHECK-NEXT: b.ne .LBB9_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: faddv d0, p1, z0.d
; CHECK-NEXT: faddv d3, p1, z3.d
-; CHECK-NEXT: faddv d1, p1, z1.d
; CHECK-NEXT: faddv d2, p1, z2.d
+; CHECK-NEXT: faddv d1, p1, z1.d
; CHECK-NEXT: mov v0.d[1], v3.d[0]
-; CHECK-NEXT: mov v1.d[1], v2.d[0]
-; CHECK-NEXT: stp q0, q1, [x2]
+; CHECK-NEXT: mov v2.d[1], v1.d[0]
+; CHECK-NEXT: stp q0, q2, [x2]
; CHECK-NEXT: ret
entry:
%vscale = tail call i64 @llvm.vscale.i64()
More information about the llvm-commits
mailing list