[llvm] [LoopUnroll] Combine partial sub/fsub reduction results with add/fadd (PR #201066)
Samuel Attard via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 2 02:04:50 PDT 2026
https://github.com/MarshallOfSound created https://github.com/llvm/llvm-project/pull/201066
When introducing parallel reduction phis during unrolling (#149470), each partial accumulator starts at the recurrence's identity value and applies the loop's updates (including any subtractions) independently, so each partial result already carries the correct sign. The final result must therefore always sum the partial results.
The combine currently uses the opcode of the recurrence kind, which is sub for `RecurKind::Sub` and fsub for `RecurKind::FSub` (reachable via #166630 with reassoc). Chaining the partial results with sub computes `pN - (... - (p1 - p0))`, flipping the sign of every other partial result and miscompiling the reduction.
Observed in the wild on AArch64 (Apple Silicon) release builds using ThinLTO and PGO, where a hand-written NEON sub-form reduction (counting bytes >= 0x80, as in simdutf's arm64 `utf8_length_from_latin1`) is hot enough to be unrolled with parallel reduction phis.
The first commit adds tests for sub/fsub reductions showing the current incorrect combines; the second applies the fix.
Fixes #201065
>From 70641239d075d02d1d76ea90895ff0275d871f1a Mon Sep 17 00:00:00 2001
From: Samuel Attard <sattard at anthropic.com>
Date: Tue, 2 Jun 2026 01:55:37 -0700
Subject: [PATCH 1/2] [LoopUnroll] Add tests for parallel reduction phis with
sub recurrences
Add tests for unrolling loops with sub and fsub reductions when adding
parallel reduction phis. The final reduction value is currently computed
by chaining the partial results with the recurrence opcode (sub/fsub),
which flips the sign of every other partial result, e.g. computing
p1 - p0 instead of p1 + p0 for an unroll count of 2. The autogenerated
checks capture the current incorrect combines; a follow-up fixes them.
---
.../LoopUnroll/runtime-unroll-reductions.ll | 142 ++++++++++++++++++
1 file changed, 142 insertions(+)
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
index 840cc6c507c3d..fcb4949bf02ad 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
@@ -484,6 +484,146 @@ exit:
ret float %res
}
+; FIXME: The combines of the partial reduction results below are incorrect:
+; partial results of sub/fsub reductions each apply their subtractions
+; starting from the identity value, so they must be combined with add/fadd.
+; The chained sub/fsub combines flip the sign of every other partial result.
+define i32 @test_sub_reduction(ptr %a, i64 %n) {
+;
+; CHECK-LABEL: define i32 @test_sub_reduction(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
+; CHECK: [[ENTRY_NEW]]:
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX_1:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 2
+; CHECK-NEXT: [[RDX_NEXT]] = sub i32 [[RDX]], [[TMP2]]
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 2
+; CHECK-NEXT: [[RDX_NEXT_1]] = sub i32 [[RDX_1]], [[TMP3]]
+; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT_UNR_LCSSA]]:
+; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT: [[BIN_RDX:%.*]] = sub i32 [[RDX_NEXT_1]], [[RDX_NEXT]]
+; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_EPIL_PREHEADER]]:
+; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
+; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
+; CHECK: [[LOOP_EPIL]]:
+; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_EPIL_INIT]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 2
+; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = sub i32 [[RDX_EPIL_INIT]], [[TMP4]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]
+ %gep.a = getelementptr inbounds nuw i32, ptr %a, i64 %iv
+ %1 = load i32, ptr %gep.a, align 2
+ %rdx.next = sub i32 %rdx, %1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop, !llvm.loop !0
+
+exit:
+ %res = phi i32 [ %rdx.next, %loop ]
+ ret i32 %res
+}
+
+define float @test_fsub_reduction(ptr %a, i64 %n) {
+;
+; CHECK-LABEL: define float @test_fsub_reduction(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
+; CHECK: [[ENTRY_NEW]]:
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX_1:%.*]] = phi float [ -0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 16
+; CHECK-NEXT: [[RDX_NEXT]] = fsub reassoc float [[RDX]], [[TMP2]]
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_NEXT]]
+; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 16
+; CHECK-NEXT: [[RDX_NEXT_1]] = fsub reassoc float [[RDX_1]], [[TMP3]]
+; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[EXIT_UNR_LCSSA]]:
+; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fsub reassoc float [[RDX_NEXT_1]], [[RDX_NEXT]]
+; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_EPIL_PREHEADER]]:
+; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
+; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
+; CHECK: [[LOOP_EPIL]]:
+; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_EPIL_INIT]]
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 16
+; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = fsub reassoc float [[RDX_EPIL_INIT]], [[TMP4]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi float [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
+; CHECK-NEXT: ret float [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi float [ 0.0, %entry ], [ %rdx.next, %loop ]
+ %gep.a = getelementptr inbounds nuw float, ptr %a, i64 %iv
+ %1 = load float, ptr %gep.a, align 16
+ %rdx.next = fsub reassoc float %rdx, %1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop, !llvm.loop !0
+
+exit:
+ %res = phi float [ %rdx.next, %loop ]
+ ret float %res
+}
+
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.unroll.count", i32 2}
@@ -500,4 +640,6 @@ exit:
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]]}
;.
>From ad3d0eb78329abdf3d2bd4b02997cdfdeb6917c8 Mon Sep 17 00:00:00 2001
From: Samuel Attard <sattard at anthropic.com>
Date: Tue, 2 Jun 2026 01:58:24 -0700
Subject: [PATCH 2/2] [LoopUnroll] Combine partial sub/fsub reduction results
with add/fadd
When introducing parallel reduction phis during unrolling, each partial
accumulator starts at the recurrence's identity value and applies the
loop's updates (including any subtractions) independently, so each
partial result already carries the correct sign. The final result must
therefore always sum the partial results.
The combine currently uses the opcode of the recurrence kind, which is
sub for RecurKind::Sub and fsub for RecurKind::FSub. Chaining the
partial results with sub computes pN - (... - (p1 - p0)), flipping the
sign of every other partial result and miscompiling the reduction.
Observed in the wild on AArch64 (Apple Silicon) release builds using
ThinLTO and PGO, where a hand-written NEON sub-form reduction (counting
bytes >= 0x80, as in simdutf's arm64 utf8_length_from_latin1) is hot
enough to be unrolled with parallel reduction phis.
Fixes #201065
---
llvm/lib/Transforms/Utils/LoopUnroll.cpp | 16 +++++++++++++---
.../LoopUnroll/runtime-unroll-reductions.ll | 8 ++------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index ffb4d1037f0f2..8d4327f0b0c27 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -1533,10 +1533,20 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
IRBuilder Builder(ExitBlock, ExitBlock->getFirstNonPHIIt());
Builder.setFastMathFlags(Reductions.begin()->second.getFastMathFlags());
RecurKind RK = Reductions.begin()->second.getRecurrenceKind();
+ // Each partial reduction starts at the recurrence's identity value
+ // and applies the loop's updates (including any subtractions)
+ // independently, so each partial result already carries the correct
+ // sign. The partial results must always be summed, even for sub-form
+ // recurrences; chaining them with the recurrence opcode (sub/fsub)
+ // would flip the sign of every other partial result.
+ unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RK);
+ if (RdxOpcode == Instruction::Sub)
+ RdxOpcode = Instruction::Add;
+ else if (RdxOpcode == Instruction::FSub)
+ RdxOpcode = Instruction::FAdd;
for (Instruction *RdxPart : drop_begin(PartialReductions)) {
- RdxResult = Builder.CreateBinOp(
- (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK),
- RdxPart, RdxResult, "bin.rdx");
+ RdxResult = Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode,
+ RdxPart, RdxResult, "bin.rdx");
}
NeedToFixLCSSA = true;
for (Instruction *RdxPart : PartialReductions)
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
index fcb4949bf02ad..40bfd86a52ef9 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
@@ -484,10 +484,6 @@ exit:
ret float %res
}
-; FIXME: The combines of the partial reduction results below are incorrect:
-; partial results of sub/fsub reductions each apply their subtractions
-; starting from the identity value, so they must be combined with add/fadd.
-; The chained sub/fsub combines flip the sign of every other partial result.
define i32 @test_sub_reduction(ptr %a, i64 %n) {
;
; CHECK-LABEL: define i32 @test_sub_reduction(
@@ -520,7 +516,7 @@ define i32 @test_sub_reduction(ptr %a, i64 %n) {
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
-; CHECK-NEXT: [[BIN_RDX:%.*]] = sub i32 [[RDX_NEXT_1]], [[RDX_NEXT]]
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add i32 [[RDX_NEXT_1]], [[RDX_NEXT]]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
@@ -588,7 +584,7 @@ define float @test_fsub_reduction(ptr %a, i64 %n) {
; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
-; CHECK-NEXT: [[BIN_RDX:%.*]] = fsub reassoc float [[RDX_NEXT_1]], [[RDX_NEXT]]
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc float [[RDX_NEXT_1]], [[RDX_NEXT]]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
More information about the llvm-commits
mailing list