[llvm] 4bf5ab4 - [VPlan] Set flags when constructing truncs using VPWidenCastRecipe.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 12 06:01:29 PDT 2025
Author: Florian Hahn
Date: 2025-10-12T14:01:12+01:00
New Revision: 4bf5ab4f9d483c6cefbbd8d04774cea5571dce45
URL: https://github.com/llvm/llvm-project/commit/4bf5ab4f9d483c6cefbbd8d04774cea5571dce45
DIFF: https://github.com/llvm/llvm-project/commit/4bf5ab4f9d483c6cefbbd8d04774cea5571dce45.diff
LOG: [VPlan] Set flags when constructing truncs using VPWidenCastRecipe.
VPWidenCastRecipes with Trunc opcodes where missing the correct OpType
for IR flags. Update createWidenCast to set the correct flags for
truncs, and use it consistenly.
Fixes https://github.com/llvm/llvm-project/issues/162374.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 456fa4c858535..7651ba16b35cf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -322,7 +322,11 @@ class VPBuilder {
VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
Type *ResultTy) {
- return tryInsertInstruction(new VPWidenCastRecipe(Opcode, Op, ResultTy));
+ VPIRFlags Flags;
+ if (Opcode == Instruction::Trunc)
+ Flags = VPIRFlags::TruncFlagsTy(false, false);
+ return tryInsertInstruction(
+ new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags));
}
VPScalarIVStepsRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 50136a8ecae57..b96d29e635465 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8774,13 +8774,19 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) &&
"Unexpected truncated min-max recurrence!");
Type *RdxTy = RdxDesc.getRecurrenceType();
- auto *Trunc =
- new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
+ VPWidenCastRecipe *Trunc;
Instruction::CastOps ExtendOpc =
RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
- auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
- Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
- Extnd->insertAfter(Trunc);
+ VPWidenCastRecipe *Extnd;
+ {
+ VPBuilder::InsertPointGuard Guard(Builder);
+ Builder.setInsertPoint(
+ NewExitingVPV->getDefiningRecipe()->getParent(),
+ std::next(NewExitingVPV->getDefiningRecipe()->getIterator()));
+ Trunc =
+ Builder.createWidenCast(Instruction::Trunc, NewExitingVPV, RdxTy);
+ Extnd = Builder.createWidenCast(ExtendOpc, Trunc, PhiTy);
+ }
if (PhiR->getOperand(1) == NewExitingVPV)
PhiR->setOperand(1, Extnd->getVPSingleValue());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7563cd719b19c..9bb88205009cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1026,6 +1026,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
PredPHI->replaceAllUsesWith(Op);
}
+ VPBuilder Builder(Def);
VPValue *A;
if (match(Def, m_Trunc(m_ZExtOrSExt(m_VPValue(A))))) {
Type *TruncTy = TypeInfo.inferScalarType(Def);
@@ -1041,18 +1042,16 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
unsigned ExtOpcode = match(R.getOperand(0), m_SExt(m_VPValue()))
? Instruction::SExt
: Instruction::ZExt;
- auto *VPC =
- new VPWidenCastRecipe(Instruction::CastOps(ExtOpcode), A, TruncTy);
+ auto *Ext = Builder.createWidenCast(Instruction::CastOps(ExtOpcode), A,
+ TruncTy);
if (auto *UnderlyingExt = R.getOperand(0)->getUnderlyingValue()) {
// UnderlyingExt has distinct return type, used to retain legacy cost.
- VPC->setUnderlyingValue(UnderlyingExt);
+ Ext->setUnderlyingValue(UnderlyingExt);
}
- VPC->insertBefore(&R);
- Def->replaceAllUsesWith(VPC);
+ Def->replaceAllUsesWith(Ext);
} else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
- auto *VPC = new VPWidenCastRecipe(Instruction::Trunc, A, TruncTy);
- VPC->insertBefore(&R);
- Def->replaceAllUsesWith(VPC);
+ auto *Trunc = Builder.createWidenCast(Instruction::Trunc, A, TruncTy);
+ Def->replaceAllUsesWith(Trunc);
}
}
#ifndef NDEBUG
@@ -1098,7 +1097,6 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
return Def->replaceAllUsesWith(Def->getOperand(1));
// (x && y) || (x && z) -> x && (y || z)
- VPBuilder Builder(Def);
if (match(Def, m_c_BinaryOr(m_LogicalAnd(m_VPValue(X), m_VPValue(Y)),
m_LogicalAnd(m_Deferred(X), m_VPValue(Z)))) &&
// Simplify only if one of the operands has one use to avoid creating an
@@ -2206,20 +2204,20 @@ void VPlanTransforms::truncateToMinimalBitwidths(
continue;
assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
auto [ProcessedIter, IterIsEmpty] = ProcessedTruncs.try_emplace(Op);
- VPWidenCastRecipe *NewOp =
- IterIsEmpty
- ? new VPWidenCastRecipe(Instruction::Trunc, Op, NewResTy,
- VPIRFlags::TruncFlagsTy(false, false))
- : ProcessedIter->second;
- R.setOperand(Idx, NewOp);
- if (!IterIsEmpty)
+ if (!IterIsEmpty) {
+ R.setOperand(Idx, ProcessedIter->second);
continue;
- ProcessedIter->second = NewOp;
- if (!Op->isLiveIn()) {
- NewOp->insertBefore(&R);
- } else {
- PH->appendRecipe(NewOp);
}
+
+ VPBuilder Builder;
+ if (Op->isLiveIn())
+ Builder.setInsertPoint(PH);
+ else
+ Builder.setInsertPoint(&R);
+ VPWidenCastRecipe *NewOp =
+ Builder.createWidenCast(Instruction::Trunc, Op, NewResTy);
+ ProcessedIter->second = NewOp;
+ R.setOperand(Idx, NewOp);
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index 000dc4a13f63a..232c354764e1a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -44,8 +44,86 @@ loop: ; preds = %loop, %entry
exit: ; preds = %loop
ret void
}
+
+; Test case for https://github.com/llvm/llvm-project/issues/162374.
+define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
+; CHECK-LABEL: define void @truncate_i16_to_i8_cse(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4294967296, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 4294967296, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 4294967296, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[SRC]], align 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i16> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[TMP8]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 8 x i8> [[TMP6]], i32 [[TMP9]]
+; CHECK-NEXT: store i8 [[TMP10]], ptr null, align 1
+; CHECK-NEXT: store i8 [[TMP10]], ptr [[DST]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 4294967296, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[COUNT:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[COUNT_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VAL:%.*]] = load i16, ptr [[SRC]], align 2
+; CHECK-NEXT: [[VAL_ZEXT:%.*]] = zext i16 [[VAL]] to i64
+; CHECK-NEXT: [[VAL_TRUNC_ZEXT:%.*]] = trunc i64 [[VAL_ZEXT]] to i8
+; CHECK-NEXT: store i8 [[VAL_TRUNC_ZEXT]], ptr null, align 1
+; CHECK-NEXT: [[VAL_TRUNC:%.*]] = trunc i16 [[VAL]] to i8
+; CHECK-NEXT: store i8 [[VAL_TRUNC]], ptr [[DST]], align 1
+; CHECK-NEXT: [[COUNT_NEXT]] = add i32 [[COUNT]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[COUNT_NEXT]], 0
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %count = phi i32 [ 0, %entry ], [ %count.next, %loop ]
+ %val = load i16, ptr %src, align 2
+ %val.zext = zext i16 %val to i64
+ %val.trunc.zext = trunc i64 %val.zext to i8
+ store i8 %val.trunc.zext, ptr null, align 1
+ %val.trunc = trunc i16 %val to i8
+ store i8 %val.trunc, ptr %dst, align 1
+ %count.next = add i32 %count, 1
+ %exitcond = icmp eq i32 %count.next, 0
+ %iv.next = add i64 %iv, 1
+ br i1 %exitcond, label %exit, label %loop
+
+exit: ; preds = %loop
+ ret void
+}
+
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
;.
More information about the llvm-commits
mailing list