[llvm] [VPlan] Fix assert when load result feeds scalar epilogue (PR #195715)
via llvm-commits
llvm-commits at lists.llvm.org
Mon May 4 11:47:21 PDT 2026
https://github.com/macurtis-amd created https://github.com/llvm/llvm-project/pull/195715
Addresses hipsolver issue noted here: https://github.com/llvm/llvm-project/pull/149706#issuecomment-4366278785.
@fhahn I'm not familiar with this code so I'm not sure if this is the right fix or not, but thought it might at least be helpful as a starting point.
I did have some assistance from AI in diagnosing the problem and creating the patch.
>From 017e58c46d2dab6a7691118af8a1b1097262a264 Mon Sep 17 00:00:00 2001
From: Matthew Curtis <macurtis at amd.com>
Date: Mon, 4 May 2026 09:21:40 -0500
Subject: [PATCH] [VPlan] Fix assert when load result feeds scalar epilogue
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 38 ++++--
...rrow-interleave-with-external-load-user.ll | 121 ++++++++++++++++++
2 files changed, 151 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vplan-narrow-interleave-with-external-load-user.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b6e9e3d5f0168..0da5b0591c260 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5239,6 +5239,25 @@ static bool isAlreadyNarrow(VPValue *VPV) {
return RepR && RepR->isSingleScalar();
}
+/// If \p R is a VPWidenLoadRecipe whose address is a VPVectorPointerRecipe,
+/// create a uniform scalar VPReplicateRecipe using the underlying scalar
+/// pointer, insert it before \p R, and return it. Returns nullptr if either
+/// cast fails.
+static VPReplicateRecipe *tryNarrowWideLoadToScalar(VPRecipeBase *R) {
+ auto *WideLoad = dyn_cast<VPWidenLoadRecipe>(R);
+ if (!WideLoad)
+ return nullptr;
+ auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(WideLoad->getAddr());
+ if (!VecPtr)
+ return nullptr;
+ auto *N =
+ new VPReplicateRecipe(&WideLoad->getIngredient(), {VecPtr->getOperand(0)},
+ /*IsUniform=*/true,
+ /*Mask=*/nullptr, {}, *WideLoad);
+ N->insertBefore(WideLoad);
+ return N;
+}
+
// Convert a wide recipe defining a VPValue \p V feeding an interleave group to
// a narrow variant.
static VPValue *
@@ -5278,16 +5297,10 @@ narrowInterleaveGroupOp(VPValue *V, SmallPtrSetImpl<VPValue *> &NarrowedOps) {
return RepR;
}
- auto *WideLoad = cast<VPWidenLoadRecipe>(R);
- VPValue *PtrOp = WideLoad->getAddr();
- if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(PtrOp))
- PtrOp = VecPtr->getOperand(0);
// Narrow wide load to uniform scalar load, as transformed VPlan will only
// process one original iteration.
- auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
- /*IsUniform*/ true,
- /*Mask*/ nullptr, {}, *WideLoad);
- N->insertBefore(WideLoad);
+ VPReplicateRecipe *N = tryNarrowWideLoadToScalar(R);
+ assert(N && "expected VPWidenLoadRecipe with VPVectorPointerRecipe addr");
NarrowedOps.insert(N);
return N;
}
@@ -5423,6 +5436,15 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
StoreGroup->eraseFromParent();
}
+ // Narrow any remaining wide loads that are not part of a store
+ // group. (e.g. whose results feed the scalar epilogue).
+ for (auto &R : make_early_inc_range(*VectorLoop->getEntryBasicBlock())) {
+ if (VPReplicateRecipe *N = tryNarrowWideLoadToScalar(&R)) {
+ cast<VPWidenLoadRecipe>(&R)->replaceAllUsesWith(N);
+ R.eraseFromParent();
+ }
+ }
+
// Adjust induction to reflect that the transformed plan only processes one
// original iteration.
VPInstruction *CanIVInc = vputils::findCanonicalIVIncrement(Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vplan-narrow-interleave-with-external-load-user.ll b/llvm/test/Transforms/LoopVectorize/X86/vplan-narrow-interleave-with-external-load-user.ll
new file mode 100644
index 0000000000000..62d9ec820c51a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vplan-narrow-interleave-with-external-load-user.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -S %s | FileCheck %s
+
+; Test that narrowInterleaveGroups correctly handles a VPWidenLoadRecipe whose
+; result is used outside the vector loop (e.g., returned via scalar epilogue),
+; when the load is not part of the interleave store group's operand tree.
+; Regression test for a crash where the VPVectorPointerRecipe addr of such a
+; load was not unwrapped, leaving it live after removeDeadRecipes and
+; triggering the assertion "All VPVectorPointerRecipes should have been removed".
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define double @widget(ptr %arg, ptr %arg1, i32 %arg2) {
+; CHECK-LABEL: define double @widget(
+; CHECK-SAME: ptr [[ARG:%.*]], ptr [[ARG1:%.*]], i32 [[ARG2:%.*]]) {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[ARG2]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
+; CHECK-NEXT: [[SMAX8:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[SMAX8]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[TMP2]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP3]], 48
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[ARG2]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 1
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 0)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[SMAX]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[SMAX]], 4294967295
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[SMAX]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[ARG]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[ARG]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ARG]], i64 8
+; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[SMAX]])
+; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp ult ptr [[TMP13]], [[SCEVGEP]]
+; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP9]], [[TMP12]]
+; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[TMP15]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP18:%.*]] = sext i32 [[ARG2]] to i64
+; CHECK-NEXT: [[TMP19:%.*]] = add nsw i64 [[TMP18]], 1
+; CHECK-NEXT: [[SMAX5:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP19]], i64 0)
+; CHECK-NEXT: [[TMP20:%.*]] = shl nuw nsw i64 [[SMAX5]], 3
+; CHECK-NEXT: [[TMP21:%.*]] = add nuw nsw i64 [[TMP20]], 8
+; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[ARG1]], i64 [[TMP21]]
+; CHECK-NEXT: [[TMP22:%.*]] = shl nuw nsw i64 [[SMAX5]], 4
+; CHECK-NEXT: [[TMP23:%.*]] = add nuw nsw i64 [[TMP22]], 16
+; CHECK-NEXT: [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[ARG]], i64 [[TMP23]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ARG1]], [[SCEVGEP7]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[ARG]], [[SCEVGEP6]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP3]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT: [[TMP25:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP26:%.*]] = sext i32 [[TMP24]] to i64
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr [8 x i8], ptr [[ARG1]], i64 [[TMP26]]
+; CHECK-NEXT: [[TMP28:%.*]] = load double, ptr [[TMP27]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr [16 x i8], ptr [[ARG]], i64 [[TMP25]]
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr [16 x i8], ptr [[ARG]], i64 [[TMP26]]
+; CHECK-NEXT: store <2 x double> splat (double 1.000000e+00), ptr [[TMP29]], align 8
+; CHECK-NEXT: store <2 x double> splat (double 1.000000e+00), ptr [[TMP30]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[BB3:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[BB]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[BB4:.*]]
+; CHECK: [[BB3]]:
+; CHECK-NEXT: [[LOAD_LCSSA:%.*]] = phi double [ [[LOAD:%.*]], %[[BB4]] ], [ [[TMP28]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret double [[LOAD_LCSSA]]
+; CHECK: [[BB4]]:
+; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[BB4]] ]
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[PHI]] to i64
+; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr [8 x i8], ptr [[ARG1]], i64 [[SEXT]]
+; CHECK-NEXT: [[LOAD]] = load double, ptr [[GETELEMENTPTR]], align 8
+; CHECK-NEXT: [[GETELEMENTPTR5:%.*]] = getelementptr [16 x i8], ptr [[ARG]], i64 [[SEXT]]
+; CHECK-NEXT: store double 1.000000e+00, ptr [[GETELEMENTPTR5]], align 8
+; CHECK-NEXT: [[GETELEMENTPTR6:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR5]], i64 8
+; CHECK-NEXT: store double 1.000000e+00, ptr [[GETELEMENTPTR6]], align 8
+; CHECK-NEXT: [[ADD]] = add i32 [[PHI]], 1
+; CHECK-NEXT: [[ICMP:%.*]] = icmp sgt i32 [[PHI]], [[ARG2]]
+; CHECK-NEXT: br i1 [[ICMP]], label %[[BB3]], label %[[BB4]], !llvm.loop [[LOOP8:![0-9]+]]
+;
+bb:
+ br label %bb4
+
+bb3: ; preds = %bb4
+ ret double %load
+
+bb4: ; preds = %bb4, %bb
+ %phi = phi i32 [ 0, %bb ], [ %add, %bb4 ]
+ %sext = sext i32 %phi to i64
+ %getelementptr = getelementptr [8 x i8], ptr %arg1, i64 %sext
+ %load = load double, ptr %getelementptr, align 8
+ %getelementptr5 = getelementptr [16 x i8], ptr %arg, i64 %sext
+ store double 1.000000e+00, ptr %getelementptr5, align 8
+ %getelementptr6 = getelementptr i8, ptr %getelementptr5, i64 8
+ store double 1.000000e+00, ptr %getelementptr6, align 8
+ %add = add i32 %phi, 1
+ %icmp = icmp sgt i32 %phi, %arg2
+ br i1 %icmp, label %bb3, label %bb4
+}
More information about the llvm-commits
mailing list