[llvm-branch-commits] [SLP]Fix alternate casts of demoted operand (PR #235280)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Oct 24 04:56:14 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
sext/sitofp of an unsigned demoted operand must be zext/uitofp, and
zext/uitofp cannot absorb the sign extension of a signed demoted operand,
so the operand must be extended back first.
Fixes https://github.com/llvm/llvm-project/issues/234923#issuecomment-6373744225
Assisted-by: Cursor
---
Full diff: https://github.com/llvm/llvm-project/pull/235280.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+98-26)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll (+8-10)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index ae9ea4c56698f2..e23d0749656781 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -16700,6 +16700,30 @@ getVectorInstrContextHint(ArrayRef<Value *> VL, const APInt &DemandedElts) {
return VIC;
}
+/// \returns the cast opcode, which gives the same result for the operand,
+/// demoted with the signedness \p IsSigned, as \p Opcode gives for the original
+/// operand, or std::nullopt if the operand must be extended back first.
+static std::optional<unsigned> getCastOpcodeForDemotedOperand(unsigned Opcode,
+ bool IsSigned) {
+ switch (Opcode) {
+ case Instruction::SExt:
+ case Instruction::SIToFP:
+ // The zero extended operand is non-negative.
+ if (IsSigned)
+ return Opcode;
+ return Opcode == Instruction::SExt ? Instruction::ZExt
+ : Instruction::UIToFP;
+ case Instruction::ZExt:
+ case Instruction::UIToFP:
+ // Cannot absorb the sign extension of the operand.
+ if (IsSigned)
+ return std::nullopt;
+ return Opcode;
+ default:
+ return std::nullopt;
+ }
+}
+
InstructionCost
BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
SmallPtrSetImpl<Value *> &CheckedExtracts) {
@@ -18195,23 +18219,39 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
cast<CmpInst>(E->getAltOp())->getPredicate(), CostKind, Op0Info,
Op1Info, E->getAltOp());
} else {
- Type *SrcSclTy = E->getMainOp()->getOperand(0)->getType();
- auto *SrcTy = getWidenedType(SrcSclTy, VL.size());
- if (SrcSclTy->isIntegerTy() && ScalarTy->isIntegerTy()) {
- auto SrcIt = MinBWs.find(getOperandEntry(E, 0));
- unsigned BWSz = DL->getTypeSizeInBits(ScalarTy);
- unsigned SrcBWSz =
- DL->getTypeSizeInBits(E->getMainOp()->getOperand(0)->getType());
- if (SrcIt != MinBWs.end()) {
- SrcBWSz = SrcIt->second.first;
- SrcSclTy = IntegerType::get(SrcSclTy->getContext(), SrcBWSz);
- SrcTy = getWidenedType(SrcSclTy, VL.size());
+ Type *SrcTy = getWidenedType(E->getMainOp()->getOperand(0)->getType(),
+ VL.size());
+ TTI::CastContextHint CCH =
+ GetCastContextHint(E->getMainOp()->getOperand(0));
+ unsigned Opcode0 = E->getOpcode();
+ unsigned Opcode1 = E->getAltOpcode();
+ // The casts of the demoted operand must take its signedness into
+ // account. If any of them cannot do it, the operand is extended back
+ // first.
+ if (auto SrcIt = MinBWs.find(getOperandEntry(E, 0));
+ SrcIt != MinBWs.end()) {
+ Type *DemotedSrcTy = SrcTy->getWithNewBitWidth(SrcIt->second.first);
+ std::optional<unsigned> DemotedOpcode0 =
+ getCastOpcodeForDemotedOperand(Opcode0, SrcIt->second.second);
+ std::optional<unsigned> DemotedOpcode1 =
+ getCastOpcodeForDemotedOperand(Opcode1, SrcIt->second.second);
+ if (DemotedOpcode0 && DemotedOpcode1) {
+ Opcode0 = *DemotedOpcode0;
+ Opcode1 = *DemotedOpcode1;
+ SrcTy = DemotedSrcTy;
+ } else {
+ VecCost += TTIRef.getCastInstrCost(
+ SrcIt->second.second ? Instruction::SExt : Instruction::ZExt,
+ SrcTy, DemotedSrcTy, CCH, CostKind);
}
+ }
+ if (SrcTy->isIntOrIntVectorTy() && ScalarTy->isIntegerTy()) {
+ unsigned BWSz = DL->getTypeSizeInBits(ScalarTy);
+ unsigned SrcBWSz = DL->getTypeSizeInBits(SrcTy->getScalarType());
if (BWSz <= SrcBWSz) {
if (BWSz < SrcBWSz)
- VecCost = TTIRef.getCastInstrCost(
- Instruction::Trunc, VecTy, SrcTy,
- GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind);
+ VecCost += TTIRef.getCastInstrCost(Instruction::Trunc, VecTy,
+ SrcTy, CCH, CostKind);
LLVM_DEBUG({
dbgs()
<< "SLP: alternate extension, which should be truncated.\n";
@@ -18220,14 +18260,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return VecCost;
}
}
- VecCost = TTIRef.getCastInstrCost(
- E->getOpcode(), VecTy, SrcTy,
- GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind,
- E->getMainOp());
- VecCost += TTIRef.getCastInstrCost(
- E->getAltOpcode(), VecTy, SrcTy,
- GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind,
- E->getAltOp());
+ // Same casts of the demoted operand, no need to shuffle.
+ if (Opcode0 == Opcode1)
+ return VecCost + TTIRef.getCastInstrCost(Opcode0, VecTy, SrcTy, CCH,
+ CostKind);
+ VecCost += TTIRef.getCastInstrCost(Opcode0, VecTy, SrcTy, CCH,
+ CostKind, E->getMainOp());
+ VecCost += TTIRef.getCastInstrCost(Opcode1, VecTy, SrcTy, CCH,
+ CostKind, E->getAltOp());
}
SmallVector<int> Mask;
E->buildAltOpShuffleMask(
@@ -25697,6 +25737,28 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
CmpInst::Predicate AltPred = AltCI->getPredicate();
V1 = Builder.CreateCmp(AltPred, LHS, RHS);
} else {
+ unsigned Opcode0 = E->getOpcode();
+ unsigned Opcode1 = E->getAltOpcode();
+ // The casts of the demoted operand must take its signedness into
+ // account. If any of them cannot do it, the operand is extended back
+ // first.
+ if (auto SrcIt = MinBWs.find(getOperandEntry(E, 0));
+ SrcIt != MinBWs.end()) {
+ std::optional<unsigned> DemotedOpcode0 =
+ getCastOpcodeForDemotedOperand(Opcode0, SrcIt->second.second);
+ std::optional<unsigned> DemotedOpcode1 =
+ getCastOpcodeForDemotedOperand(Opcode1, SrcIt->second.second);
+ if (DemotedOpcode0 && DemotedOpcode1) {
+ Opcode0 = *DemotedOpcode0;
+ Opcode1 = *DemotedOpcode1;
+ } else {
+ LHS = Builder.CreateIntCast(
+ LHS,
+ getWidenedType(E->getMainOp()->getOperand(0)->getType(),
+ E->Scalars.size()),
+ SrcIt->second.second);
+ }
+ }
if (LHS->getType()->isIntOrIntVectorTy() && ScalarTy->isIntegerTy()) {
unsigned SrcBWSz = DL->getTypeSizeInBits(
cast<VectorType>(LHS->getType())->getElementType());
@@ -25713,10 +25775,20 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return LHS;
}
}
- V0 = Builder.CreateCast(
- static_cast<Instruction::CastOps>(E->getOpcode()), LHS, VecTy);
- V1 = Builder.CreateCast(
- static_cast<Instruction::CastOps>(E->getAltOpcode()), LHS, VecTy);
+ // Same casts of the demoted operand, no need to shuffle.
+ if (Opcode0 == Opcode1) {
+ V = Builder.CreateCast(static_cast<Instruction::CastOps>(Opcode0),
+ LHS, VecTy);
+ V = FinalShuffle(V, E);
+ E->VectorizedValue = V;
+ ++NumVectorInstructions;
+ return V;
+ }
+ V0 = Builder.CreateCast(static_cast<Instruction::CastOps>(Opcode0),
+ LHS, VecTy);
+ V1 = Builder.CreateCast(static_cast<Instruction::CastOps>(Opcode1),
+ LHS, VecTy);
+ NumVectorInstructions += 2;
}
// Add V0 and V1 to later analysis to try to find and remove matching
// instruction, if any.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll
index b45037e1be67a1..4c257a43e19307 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll
@@ -9,11 +9,9 @@ define void @alt_zext_sext_unsigned_demoted_operand(ptr noalias %p, ptr noalias
; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
-; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <4 x i8> [[TMP0]], splat (i8 -6)
; CHECK-NEXT: [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i32>
-; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP1]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i32> [[TMP6]], splat (i32 16843009)
; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[PQ0]], align 4
; CHECK-NEXT: ret void
@@ -69,8 +67,9 @@ define void @alt_sext_zext_signed_demoted_operand(ptr noalias %p, ptr noalias %q
; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0
; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32>
-; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i16> [[TMP8]] to <4 x i32>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <4 x i8> [[TMP0]], splat (i8 100)
; CHECK-NEXT: [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i32>
@@ -131,9 +130,7 @@ define void @alt_uitofp_sitofp_unsigned_demoted_operand(ptr noalias %p, ptr noal
; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = uitofp <4 x i8> [[TMP0]] to <4 x float>
-; CHECK-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP0]] to <4 x float>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <4 x i8> [[TMP0]], splat (i8 -6)
; CHECK-NEXT: [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32>
; CHECK-NEXT: [[TMP7:%.*]] = and <4 x i32> [[TMP4]], [[TMP6]]
@@ -191,8 +188,9 @@ define void @alt_sitofp_uitofp_signed_demoted_operand(ptr noalias %p, ptr noalia
; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0
; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i8> [[TMP0]] to <4 x float>
-; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i8> [[TMP0]] to <4 x float>
+; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP8]] to <4 x float>
+; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i16> [[TMP8]] to <4 x float>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float> [[TMP3]] to <4 x i32>
; CHECK-NEXT: [[TMP5:%.*]] = icmp slt <4 x i8> [[TMP0]], splat (i8 100)
``````````
</details>
https://github.com/llvm/llvm-project/pull/235280
More information about the llvm-branch-commits
mailing list