[llvm] [SLP] Allow UDiv X, C <--> LShr X, log2(C) transformations in BinOpSameOpcodeHelper (PR #181731)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 11 20:24:56 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/181731
>From 031ffc87239a77d9e302cf3edbc5403ce8ebd285 Mon Sep 17 00:00:00 2001
From: bababuck <rbuchner at qti.qualcomm.com>
Date: Sun, 8 Feb 2026 09:56:09 -0800
Subject: [PATCH 01/15] [SLP] Add tests for LShr-UDiv power of 2 vectorization
---
.../SLPVectorizer/semanticly-same.ll | 318 +++++++++++++++++-
1 file changed, 316 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
index 4e0d22400aeb8..ac0a219c9ae3b 100644
--- a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
+++ b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
@@ -1,10 +1,78 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s --check-prefixes=CHECK,X86 %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s --check-prefixes=CHECK,AARCH64 %}
; Don't care about the profitability with these tests, just want to demonstrate the ability
; to combine opcodes
+define void @shl_add(ptr %p, ptr %s) {
+; CHECK-LABEL: @shl_add(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i16> [[TMP0]], <i16 3, i16 5, i16 0, i16 3>
+; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %shl0 = shl i16 %l0, 3
+ %shl1 = shl i16 %l1, 5
+ %shl2 = add i16 %l2, 0
+ %shl3 = shl i16 %l3, 3
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %shl0, ptr %s
+ store i16 %shl1, ptr %s1
+ store i16 %shl2, ptr %s2
+ store i16 %shl3, ptr %s3
+ ret void
+}
+
+define void @add_shl(ptr %p, ptr %s) {
+; CHECK-LABEL: @add_shl(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i16> [[TMP0]], <i16 0, i16 5, i16 2, i16 3>
+; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %add0 = shl i16 %l0, 0
+ %add1 = add i16 %l1, 5
+ %add2 = add i16 %l2, 2
+ %add3 = add i16 %l3, 3
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %add0, ptr %s
+ store i16 %add1, ptr %s1
+ store i16 %add2, ptr %s2
+ store i16 %add3, ptr %s3
+ ret void
+}
+
define void @sub_mul(ptr %p, ptr %s) {
; CHECK-LABEL: @sub_mul(
; CHECK-NEXT: entry:
@@ -106,3 +174,249 @@ entry:
store i16 %mul3, ptr %s3
ret void
}
+
+define void @shl_mul(ptr %p, ptr %s) {
+; CHECK-LABEL: @shl_mul(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = mul <4 x i16> [[TMP0]], <i16 8, i16 5, i16 2, i16 3>
+; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %mul0 = shl i16 %l0, 3
+ %mul1 = mul i16 %l1, 5
+ %mul2 = mul i16 %l2, 2
+ %mul3 = mul i16 %l3, 3
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %mul0, ptr %s
+ store i16 %mul1, ptr %s1
+ store i16 %mul2, ptr %s2
+ store i16 %mul3, ptr %s3
+ ret void
+}
+
+define void @lshr_div(ptr %p, ptr %s) {
+; CHECK-LABEL: @lshr_div(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i16> [[TMP0]], <i16 3, i16 0, i16 0, i16 0>
+; CHECK-NEXT: [[TMP1:%.*]] = udiv <4 x i16> [[TMP2]], <i16 1, i16 5, i16 2, i16 3>
+; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %div0 = lshr i16 %l0, 3
+ %div1 = udiv i16 %l1, 5
+ %div2 = udiv i16 %l2, 2
+ %div3 = udiv i16 %l3, 3
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %div0, ptr %s
+ store i16 %div1, ptr %s1
+ store i16 %div2, ptr %s2
+ store i16 %div3, ptr %s3
+ ret void
+}
+
+define void @div_lshr(ptr %p, ptr %s) {
+; CHECK-LABEL: @div_lshr(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP0]], <i16 5, i16 1, i16 2, i16 1>
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP2]], <i16 0, i16 3, i16 0, i16 9>
+; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %div0 = udiv i16 %l0, 5
+ %div1 = lshr i16 %l1, 3
+ %div2 = udiv i16 %l2, 2
+ %div3 = lshr i16 %l3, 9
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %div0, ptr %s
+ store i16 %div1, ptr %s1
+ store i16 %div2, ptr %s2
+ store i16 %div3, ptr %s3
+ ret void
+}
+
+define void @div_lshr_too_large(ptr %p, ptr %s) {
+; CHECK-LABEL: @div_lshr_too_large(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = udiv <4 x i16> [[TMP0]], <i16 5, i16 1, i16 2, i16 1>
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i16> [[TMP1]], <i16 0, i16 17, i16 0, i16 9>
+; CHECK-NEXT: store <4 x i16> [[TMP2]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %div0 = udiv i16 %l0, 5
+ %div1 = lshr i16 %l1, 17
+ %div2 = udiv i16 %l2, 2
+ %div3 = lshr i16 %l3, 9
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %div0, ptr %s
+ store i16 %div1, ptr %s1
+ store i16 %div2, ptr %s2
+ store i16 %div3, ptr %s3
+ ret void
+}
+
+define void @lshr_div2(ptr %p, ptr %s) {
+; CHECK-LABEL: @lshr_div2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP0]], <i16 3, i16 0, i16 0, i16 5>
+; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP1]], <i16 1, i16 8, i16 2, i16 1>
+; CHECK-NEXT: store <4 x i16> [[TMP2]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %div0 = lshr i16 %l0, 3
+ %div1 = udiv i16 %l1, 8
+ %div2 = udiv i16 %l2, 2
+ %div3 = lshr i16 %l3, 5
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %div0, ptr %s
+ store i16 %div1, ptr %s1
+ store i16 %div2, ptr %s2
+ store i16 %div3, ptr %s3
+ ret void
+}
+
+define void @div2_lshr(ptr %p, ptr %s) {
+; CHECK-LABEL: @div2_lshr(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP0]], <i16 8, i16 1, i16 4, i16 1>
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP2]], <i16 0, i16 3, i16 0, i16 4>
+; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %div0 = udiv i16 %l0, 8
+ %div1 = lshr i16 %l1, 3
+ %div2 = udiv i16 %l2, 4
+ %div3 = lshr i16 %l3, 4
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %div0, ptr %s
+ store i16 %div1, ptr %s1
+ store i16 %div2, ptr %s2
+ store i16 %div3, ptr %s3
+ ret void
+}
+
+define void @add_sub(ptr %p, ptr %s) {
+; CHECK-LABEL: @add_sub(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i16> [[TMP0]], <i16 3, i16 5, i16 2, i16 3>
+; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %p1 = getelementptr i16, ptr %p, i64 1
+ %p2 = getelementptr i16, ptr %p, i64 2
+ %p3 = getelementptr i16, ptr %p, i64 3
+
+ %l0 = load i16, ptr %p
+ %l1 = load i16, ptr %p1
+ %l2 = load i16, ptr %p2
+ %l3 = load i16, ptr %p3
+
+ %add0 = sub i16 %l0, -3
+ %add1 = add i16 %l1, 5
+ %add2 = add i16 %l2, 2
+ %add3 = add i16 %l3, 3
+
+ %s1 = getelementptr i16, ptr %s, i64 1
+ %s2 = getelementptr i16, ptr %s, i64 2
+ %s3 = getelementptr i16, ptr %s, i64 3
+
+ store i16 %add0, ptr %s
+ store i16 %add1, ptr %s1
+ store i16 %add2, ptr %s2
+ store i16 %add3, ptr %s3
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AARCH64: {{.*}}
+; X86: {{.*}}
>From ada80af6767296d2a2cc54c39f9fbfb572eff9aa Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Fri, 13 Feb 2026 16:34:49 -0800
Subject: [PATCH 02/15] [SLP] Support LShr in BinOpSameOpcodeHelper
LShr X, 0 is an identity.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 24 +++++++++-----
.../RISCV/partial-vec-invalid-cost.ll | 21 +++++--------
.../X86/matching-insert-point-for-nodes.ll | 10 +++---
.../X86/non-power-of-2-subvectors-insert.ll | 7 +++--
.../X86/reschedule-only-scheduled.ll | 26 +++++++++-------
.../SLPVectorizer/semanticly-same.ll | 31 ++++++++++++++-----
6 files changed, 69 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7ac5dea2b3d5f..98d588428f4d4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -952,8 +952,9 @@ class BinOpSameOpcodeHelper {
using MaskType = std::uint_fast16_t;
/// Sort SupportedOp because it is used by binary_search.
constexpr static std::initializer_list<unsigned> SupportedOp = {
- Instruction::Add, Instruction::Sub, Instruction::Mul, Instruction::Shl,
- Instruction::AShr, Instruction::And, Instruction::Or, Instruction::Xor};
+ Instruction::Add, Instruction::Sub, Instruction::Mul,
+ Instruction::Shl, Instruction::LShr, Instruction::AShr,
+ Instruction::And, Instruction::Or, Instruction::Xor};
static_assert(llvm::is_sorted_constexpr(SupportedOp) &&
"SupportedOp is not sorted.");
enum : MaskType {
@@ -965,7 +966,8 @@ class BinOpSameOpcodeHelper {
AndBIT = 0b100000,
OrBIT = 0b1000000,
XorBIT = 0b10000000,
- MainOpBIT = 0b100000000,
+ LShrBIT = 0b100000000,
+ MainOpBIT = 0b1000000000,
LLVM_MARK_AS_BITMASK_ENUM(MainOpBIT)
};
/// Return a non-nullptr if either operand of I is a ConstantInt.
@@ -982,7 +984,7 @@ class BinOpSameOpcodeHelper {
if (auto *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)))
return {CI, 1};
if (Opcode == Instruction::Sub || Opcode == Instruction::Shl ||
- Opcode == Instruction::AShr)
+ Opcode == Instruction::AShr || Opcode == Instruction::LShr)
return {nullptr, 0};
if (auto *CI = dyn_cast<ConstantInt>(BinOp->getOperand(0)))
return {CI, 0};
@@ -992,7 +994,7 @@ class BinOpSameOpcodeHelper {
const Instruction *I = nullptr;
/// The bit it sets represents whether MainOp can be converted to.
MaskType Mask = MainOpBIT | XorBIT | OrBIT | AndBIT | SubBIT | AddBIT |
- MulBIT | AShrBIT | ShlBIT;
+ MulBIT | AShrBIT | ShlBIT | LShrBIT;
/// We cannot create an interchangeable instruction that does not exist in
/// VL. For example, VL [x + 0, y * 1] can be converted to [x << 0, y << 0],
/// but << does not exist in VL. In the end, we convert VL to [x * 1, y *
@@ -1033,6 +1035,8 @@ class BinOpSameOpcodeHelper {
return Instruction::Or;
if (Candidate & XorBIT)
return Instruction::Xor;
+ if (Candidate & LShrBIT)
+ return Instruction::LShr;
llvm_unreachable("Cannot find interchangeable instruction.");
}
@@ -1057,6 +1061,7 @@ class BinOpSameOpcodeHelper {
case Instruction::Xor:
return Candidate & XorBIT;
case Instruction::LShr:
+ return Candidate & LShrBIT;
case Instruction::FAdd:
case Instruction::FSub:
case Instruction::FMul:
@@ -1166,7 +1171,7 @@ class BinOpSameOpcodeHelper {
"BinOpSameOpcodeHelper only accepts BinaryOperator.");
unsigned Opcode = I->getOpcode();
MaskType OpcodeInMaskForm;
- // Prefer Shl, AShr, Mul, Add, Sub, And, Or and Xor over MainOp.
+ // Prefer Shl, AShr, Mul, Add, Sub, And, Or, Xor, and LShr over MainOp.
switch (Opcode) {
case Instruction::Shl:
OpcodeInMaskForm = ShlBIT;
@@ -1192,6 +1197,9 @@ class BinOpSameOpcodeHelper {
case Instruction::Xor:
OpcodeInMaskForm = XorBIT;
break;
+ case Instruction::LShr:
+ OpcodeInMaskForm = LShrBIT;
+ break;
default:
return MainOp.equal(Opcode) ||
(initializeAltOp(I) && AltOp.equal(Opcode));
@@ -1199,8 +1207,8 @@ class BinOpSameOpcodeHelper {
MaskType InterchangeableMask = OpcodeInMaskForm;
ConstantInt *CI = isBinOpWithConstantInt(I).first;
if (CI) {
- constexpr MaskType CanBeAll =
- XorBIT | OrBIT | AndBIT | SubBIT | AddBIT | MulBIT | AShrBIT | ShlBIT;
+ constexpr MaskType CanBeAll = XorBIT | OrBIT | AndBIT | SubBIT | AddBIT |
+ MulBIT | AShrBIT | ShlBIT | LShrBIT;
const APInt &CIValue = CI->getValue();
switch (Opcode) {
case Instruction::Shl:
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
index 2b79ca9429fa3..3c6ea5a7cce87 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
@@ -7,25 +7,18 @@ define void @partial_vec_invalid_cost() #0 {
; CHECK-LABEL: define void @partial_vec_invalid_cost(
; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LSHR_1:%.*]] = lshr i96 0, 0
-; CHECK-NEXT: [[LSHR_2:%.*]] = lshr i96 0, 0
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i96> poison, i96 [[LSHR_1]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i96> [[TMP0]], i96 [[LSHR_2]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i96> [[TMP1]], i96 0, i32 2
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i96> [[TMP2]], i96 0, i32 3
-; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i96> [[TMP3]] to <4 x i32>
-; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i32> zeroinitializer, [[TMP4]]
-; CHECK-NEXT: [[OP_RDX3:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[RDX_OP]])
+; CHECK-NEXT: [[OP_RDX3:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer)
; CHECK-NEXT: [[STORE_THIS:%.*]] = zext i32 [[OP_RDX3]] to i96
; CHECK-NEXT: store i96 [[STORE_THIS]], ptr null, align 16
; CHECK-NEXT: ret void
;
entry:
-
- %lshr.1 = lshr i96 0, 0 ; These ops
- %lshr.2 = lshr i96 0, 0 ; return an
- %add.0 = add i96 0, 0 ; invalid
- %add.1 = add i96 0, 0 ; vector cost.
+ ; Test is broken, I don't think there are any pairs of binary ops that
+ ; can give an invalid cost
+ %lshr.1 = lshr i96 0, 0
+ %lshr.2 = lshr i96 0, 0
+ %add.0 = add i96 0, 0
+ %add.1 = add i96 0, 0
%trunc.i96.1 = trunc i96 %lshr.1 to i32 ; These ops
%trunc.i96.2 = trunc i96 %lshr.2 to i32 ; return an
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/matching-insert-point-for-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/matching-insert-point-for-nodes.ll
index 5e85ecd610ebd..8a6705295df81 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/matching-insert-point-for-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/matching-insert-point-for-nodes.ll
@@ -6,8 +6,8 @@ define i32 @test() {
; CHECK-NEXT: [[BB:.*]]:
; CHECK-NEXT: br label %[[BB1:.*]]
; CHECK: [[BB1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ [[TMP16:%.*]], %[[BB24:.*]] ], [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i32> [ [[TMP17:%.*]], %[[BB24]] ], [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ [[TMP14:%.*]], %[[BB24:.*]] ], [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i32> [ [[TMP13:%.*]], %[[BB24]] ], [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB]] ]
; CHECK-NEXT: br i1 false, label %[[BB4:.*]], label %[[BB11:.*]]
; CHECK: [[BB4]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi <2 x double> [ zeroinitializer, %[[BB1]] ]
@@ -33,10 +33,8 @@ define i32 @test() {
; CHECK-NEXT: [[TMP9:%.*]] = lshr <4 x i32> [[TMP8]], <i32 poison, i32 poison, i32 0, i32 0>
; CHECK-NEXT: [[TMP10:%.*]] = and <4 x i32> [[TMP9]], <i32 poison, i32 poison, i32 0, i32 -1>
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
-; CHECK-NEXT: [[TMP14:%.*]] = lshr <4 x i32> [[TMP11]], [[TMP10]]
-; CHECK-NEXT: [[TMP15:%.*]] = or <4 x i32> [[TMP11]], [[TMP10]]
-; CHECK-NEXT: [[TMP16]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> [[TMP15]], <4 x i32> <i32 poison, i32 poison, i32 2, i32 7>
-; CHECK-NEXT: [[TMP17]] = shufflevector <4 x i32> [[TMP16]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, <4 x i32> <i32 poison, i32 poison, i32 2, i32 7>
+; CHECK-NEXT: [[TMP14]] = lshr <4 x i32> [[TMP11]], [[TMP10]]
+; CHECK-NEXT: [[TMP13]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
; CHECK-NEXT: br label %[[BB1]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
index 3e9bd781bfea1..d85d5a61df53b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
@@ -3,9 +3,10 @@
define void @test() {
; CHECK-LABEL: define void @test() {
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i64> poison, i64 1, i32 3
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 1, i64 0, i64 undef, i64 undef, i64 undef, i64 undef>, <16 x i64> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i64> [[TMP6]], <16 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 3, i32 7, i32 8, i32 9, i32 3, i32 10, i32 11, i32 12, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <14 x i64> <i64 undef, i64 undef, i64 0, i64 1, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 0, i64 undef>, <14 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 1, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>, <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <14 x i64> [[TMP1]], <14 x i64> <i64 0, i64 0, i64 0, i64 0, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>, <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 14, i32 15, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <14 x i64> [[TMP2]], <14 x i64> <i64 0, i64 0, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>, <14 x i32> <i32 14, i32 15, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <14 x i64> [[TMP3]], <14 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 3, i32 7, i32 8, i32 9, i32 3, i32 10, i32 11, i32 12, i32 3>
; CHECK-NEXT: [[TMP8:%.*]] = trunc <16 x i64> [[TMP7]] to <16 x i1>
; CHECK-NEXT: [[TMP9:%.*]] = or <16 x i1> [[TMP8]], zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = freeze <16 x i1> [[TMP9]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reschedule-only-scheduled.ll b/llvm/test/Transforms/SLPVectorizer/X86/reschedule-only-scheduled.ll
index 2dd6b395597c3..1301907b2e032 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reschedule-only-scheduled.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reschedule-only-scheduled.ll
@@ -5,20 +5,22 @@ define i16 @test() {
; CHECK-LABEL: define i16 @test() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = lshr i32 0, 0
-; CHECK-NEXT: [[TMP10:%.*]] = shl i32 [[TMP0]], 0
; CHECK-NEXT: [[CALL99_I:%.*]] = call i32 @llvm.bswap.i32(i32 0)
-; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[CALL99_I]], 0
; CHECK-NEXT: [[CALL7_I45:%.*]] = tail call i32 null(i32 0)
-; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[CALL7_I45]], 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <28 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 poison>, i32 [[TMP10]], i32 4
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <28 x i32> [[TMP4]], i32 [[TMP2]], i32 5
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <28 x i32> [[TMP5]], <28 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 28, i32 29, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <28 x i32> [[TMP6]], i32 [[TMP8]], i32 12
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <28 x i32> [[TMP7]], <28 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <28 x i32> [[TMP16]], <28 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 28, i32 29, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <28 x i32> [[TMP9]], <28 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 28, i32 29, i32 30, i32 31, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: [[TMP11:%.*]] = and <28 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison>, [[TMP17]]
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <28 x i32> [[TMP11]], <28 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 6, i32 7, i32 7, i32 8, i32 9, i32 10, i32 10, i32 11, i32 12, i32 13, i32 13, i32 14, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 0, 0
+; CHECK-NEXT: [[TMP2:%.*]] = shl i32 0, 0
+; CHECK-NEXT: [[UNSCLEAR78_I:%.*]] = and i32 [[TMP2]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <12 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 0, i32 0>, i32 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <12 x i32> [[TMP3]], i32 [[CALL99_I]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <12 x i32> [[TMP4]], i32 [[CALL7_I45]], i32 8
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <12 x i32> [[TMP5]], i32 [[TMP1]], i32 9
+; CHECK-NEXT: [[TMP7:%.*]] = shl <12 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = and <12 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <12 x i32> [[TMP8]], <12 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 poison, i32 poison, i32 poison, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <32 x i32> [[TMP10]], i32 [[UNSCLEAR78_I]], i32 16
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <32 x i32> [[TMP11]], <32 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 32, i32 33, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <32 x i32> [[TMP16]], <32 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 6, i32 7, i32 7, i32 8, i32 9, i32 10, i32 10, i32 11, i32 12, i32 13, i32 13, i32 14, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
; CHECK-NEXT: [[TMP13:%.*]] = icmp ne <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1>, [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = bitcast <32 x i1> [[TMP13]] to i32
; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.ctpop.i32(i32 [[TMP14]])
diff --git a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
index ac0a219c9ae3b..4e7a8270bde78 100644
--- a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
+++ b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
@@ -315,13 +315,30 @@ entry:
}
define void @lshr_div2(ptr %p, ptr %s) {
-; CHECK-LABEL: @lshr_div2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP0]], <i16 3, i16 0, i16 0, i16 5>
-; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP1]], <i16 1, i16 8, i16 2, i16 1>
-; CHECK-NEXT: store <4 x i16> [[TMP2]], ptr [[S:%.*]], align 2
-; CHECK-NEXT: ret void
+; X86-LABEL: @lshr_div2(
+; X86-NEXT: entry:
+; X86-NEXT: [[P1:%.*]] = getelementptr i16, ptr [[P:%.*]], i64 1
+; X86-NEXT: [[P3:%.*]] = getelementptr i16, ptr [[P]], i64 3
+; X86-NEXT: [[L0:%.*]] = load i16, ptr [[P]], align 2
+; X86-NEXT: [[L3:%.*]] = load i16, ptr [[P3]], align 2
+; X86-NEXT: [[DIV0:%.*]] = lshr i16 [[L0]], 3
+; X86-NEXT: [[DIV3:%.*]] = lshr i16 [[L3]], 5
+; X86-NEXT: [[S1:%.*]] = getelementptr i16, ptr [[S:%.*]], i64 1
+; X86-NEXT: [[S3:%.*]] = getelementptr i16, ptr [[S]], i64 3
+; X86-NEXT: [[TMP0:%.*]] = load <2 x i16>, ptr [[P1]], align 2
+; X86-NEXT: [[TMP1:%.*]] = udiv <2 x i16> [[TMP0]], <i16 8, i16 2>
+; X86-NEXT: store i16 [[DIV0]], ptr [[S]], align 2
+; X86-NEXT: store <2 x i16> [[TMP1]], ptr [[S1]], align 2
+; X86-NEXT: store i16 [[DIV3]], ptr [[S3]], align 2
+; X86-NEXT: ret void
+;
+; AARCH64-LABEL: @lshr_div2(
+; AARCH64-NEXT: entry:
+; AARCH64-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; AARCH64-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP0]], <i16 3, i16 0, i16 0, i16 5>
+; AARCH64-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP1]], <i16 1, i16 8, i16 2, i16 1>
+; AARCH64-NEXT: store <4 x i16> [[TMP2]], ptr [[S:%.*]], align 2
+; AARCH64-NEXT: ret void
;
entry:
%p1 = getelementptr i16, ptr %p, i64 1
>From edf66fb7bca293b3849e9e242560344450be87f8 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Fri, 13 Feb 2026 16:49:49 -0800
Subject: [PATCH 03/15] [SLP] Support UDiv in BinOpSameOpcodeHelper
UDiv X, 1 is an identity
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 29 +++++++++++++------
.../SLPVectorizer/X86/no_alternate_divrem.ll | 19 ++++++++----
2 files changed, 33 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 98d588428f4d4..f1f87b9a62815 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -952,9 +952,9 @@ class BinOpSameOpcodeHelper {
using MaskType = std::uint_fast16_t;
/// Sort SupportedOp because it is used by binary_search.
constexpr static std::initializer_list<unsigned> SupportedOp = {
- Instruction::Add, Instruction::Sub, Instruction::Mul,
- Instruction::Shl, Instruction::LShr, Instruction::AShr,
- Instruction::And, Instruction::Or, Instruction::Xor};
+ Instruction::Add, Instruction::Sub, Instruction::Mul, Instruction::UDiv,
+ Instruction::Shl, Instruction::LShr, Instruction::AShr, Instruction::And,
+ Instruction::Or, Instruction::Xor};
static_assert(llvm::is_sorted_constexpr(SupportedOp) &&
"SupportedOp is not sorted.");
enum : MaskType {
@@ -967,7 +967,8 @@ class BinOpSameOpcodeHelper {
OrBIT = 0b1000000,
XorBIT = 0b10000000,
LShrBIT = 0b100000000,
- MainOpBIT = 0b1000000000,
+ UDivBIT = 0b1000000000,
+ MainOpBIT = 0b10000000000,
LLVM_MARK_AS_BITMASK_ENUM(MainOpBIT)
};
/// Return a non-nullptr if either operand of I is a ConstantInt.
@@ -984,7 +985,8 @@ class BinOpSameOpcodeHelper {
if (auto *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)))
return {CI, 1};
if (Opcode == Instruction::Sub || Opcode == Instruction::Shl ||
- Opcode == Instruction::AShr || Opcode == Instruction::LShr)
+ Opcode == Instruction::AShr || Opcode == Instruction::LShr ||
+ Opcode == Instruction::UDiv)
return {nullptr, 0};
if (auto *CI = dyn_cast<ConstantInt>(BinOp->getOperand(0)))
return {CI, 0};
@@ -994,7 +996,7 @@ class BinOpSameOpcodeHelper {
const Instruction *I = nullptr;
/// The bit it sets represents whether MainOp can be converted to.
MaskType Mask = MainOpBIT | XorBIT | OrBIT | AndBIT | SubBIT | AddBIT |
- MulBIT | AShrBIT | ShlBIT | LShrBIT;
+ MulBIT | AShrBIT | ShlBIT | LShrBIT | UDivBIT;
/// We cannot create an interchangeable instruction that does not exist in
/// VL. For example, VL [x + 0, y * 1] can be converted to [x << 0, y << 0],
/// but << does not exist in VL. In the end, we convert VL to [x * 1, y *
@@ -1037,6 +1039,8 @@ class BinOpSameOpcodeHelper {
return Instruction::Xor;
if (Candidate & LShrBIT)
return Instruction::LShr;
+ if (Candidate & UDivBIT)
+ return Instruction::UDiv;
llvm_unreachable("Cannot find interchangeable instruction.");
}
@@ -1062,11 +1066,12 @@ class BinOpSameOpcodeHelper {
return Candidate & XorBIT;
case Instruction::LShr:
return Candidate & LShrBIT;
+ case Instruction::UDiv:
+ return Candidate & UDivBIT;
case Instruction::FAdd:
case Instruction::FSub:
case Instruction::FMul:
case Instruction::SDiv:
- case Instruction::UDiv:
case Instruction::FDiv:
case Instruction::SRem:
case Instruction::URem:
@@ -1102,6 +1107,7 @@ class BinOpSameOpcodeHelper {
}
break;
case Instruction::Mul:
+ case Instruction::UDiv:
assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
if (ToOpcode == Instruction::Shl) {
RHS = ConstantInt::get(
@@ -1171,7 +1177,8 @@ class BinOpSameOpcodeHelper {
"BinOpSameOpcodeHelper only accepts BinaryOperator.");
unsigned Opcode = I->getOpcode();
MaskType OpcodeInMaskForm;
- // Prefer Shl, AShr, Mul, Add, Sub, And, Or, Xor, and LShr over MainOp.
+ // Prefer Shl, AShr, Mul, Add, Sub, And, Or, Xor, LShr, and UDiv over
+ // MainOp.
switch (Opcode) {
case Instruction::Shl:
OpcodeInMaskForm = ShlBIT;
@@ -1200,6 +1207,9 @@ class BinOpSameOpcodeHelper {
case Instruction::LShr:
OpcodeInMaskForm = LShrBIT;
break;
+ case Instruction::UDiv:
+ OpcodeInMaskForm = UDivBIT;
+ break;
default:
return MainOp.equal(Opcode) ||
(initializeAltOp(I) && AltOp.equal(Opcode));
@@ -1208,7 +1218,8 @@ class BinOpSameOpcodeHelper {
ConstantInt *CI = isBinOpWithConstantInt(I).first;
if (CI) {
constexpr MaskType CanBeAll = XorBIT | OrBIT | AndBIT | SubBIT | AddBIT |
- MulBIT | AShrBIT | ShlBIT | LShrBIT;
+ MulBIT | AShrBIT | ShlBIT | LShrBIT |
+ UDivBIT;
const APInt &CIValue = CI->getValue();
switch (Opcode) {
case Instruction::Shl:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll b/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
index a888027479817..edb69f5ca8293 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
@@ -51,14 +51,21 @@ entry:
define void @test_add_udiv(ptr %arr1, ptr %arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
; CHECK-LABEL: @test_add_udiv(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[ARR1:%.*]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[A2:%.*]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], <i32 1, i32 1, i32 42, i32 1>
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[A0:%.*]], i32 0
+; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 2
+; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr i32, ptr [[ARR1]], i32 3
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[GEP1_2]], align 4
+; CHECK-NEXT: [[V3:%.*]] = load i32, ptr [[GEP1_3]], align 4
+; CHECK-NEXT: [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[ARR1]], align 4
+; CHECK-NEXT: [[RES2:%.*]] = udiv i32 [[V2]], [[Y2]]
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[A0:%.*]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A1:%.*]], i32 1
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[A3:%.*]], i32 3
-; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> <i32 1146, i32 146, i32 0, i32 0>, [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = udiv <4 x i32> [[TMP0]], [[TMP6]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[RES2]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1146, i32 146, i32 0, i32 0>
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[V3]], i32 3
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; CHECK-NEXT: [[TMP9:%.*]] = add nsw <4 x i32> [[TMP8]], [[TMP4]]
; CHECK-NEXT: store <4 x i32> [[TMP9]], ptr [[ARR2:%.*]], align 4
; CHECK-NEXT: ret void
>From 68ccbb1ded400c5abf6124c4a156cfd42d006983 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Fri, 13 Feb 2026 16:52:59 -0800
Subject: [PATCH 04/15] [SLP] Allow UDiv X, C <--> LShr X, log2(C)
tranformations in BinOpSameOpcodeHelper
When C is a power of 2.
Important since InstCombine may convert a UDiv to LShr.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 18 ++++++--
.../SLPVectorizer/semanticly-same.ll | 43 +++++--------------
2 files changed, 25 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f1f87b9a62815..c02a3bd7d3f10 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1096,7 +1096,8 @@ class BinOpSameOpcodeHelper {
Constant *RHS;
switch (FromOpcode) {
case Instruction::Shl:
- if (ToOpcode == Instruction::Mul) {
+ case Instruction::LShr:
+ if (ToOpcode == Instruction::Mul || ToOpcode == Instruction::UDiv) {
RHS = ConstantInt::get(
RHSType, APInt::getOneBitSet(FromCIValueBitWidth,
FromCIValue.getZExtValue()));
@@ -1109,7 +1110,7 @@ class BinOpSameOpcodeHelper {
case Instruction::Mul:
case Instruction::UDiv:
assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
- if (ToOpcode == Instruction::Shl) {
+ if (ToOpcode == Instruction::Shl || ToOpcode == Instruction::LShr) {
RHS = ConstantInt::get(
RHSType, APInt(FromCIValueBitWidth, FromCIValue.logBase2()));
} else {
@@ -1226,13 +1227,22 @@ class BinOpSameOpcodeHelper {
if (CIValue.ult(CIValue.getBitWidth()))
InterchangeableMask = CIValue.isZero() ? CanBeAll : MulBIT | ShlBIT;
break;
+ case Instruction::LShr:
+ if (CIValue.ult(CIValue.getBitWidth()))
+ InterchangeableMask = CIValue.isZero() ? CanBeAll : UDivBIT | LShrBIT;
+ break;
case Instruction::Mul:
+ case Instruction::UDiv:
if (CIValue.isOne()) {
InterchangeableMask = CanBeAll;
break;
}
- if (CIValue.isPowerOf2())
- InterchangeableMask = MulBIT | ShlBIT;
+ if (CIValue.isPowerOf2()) {
+ if (Opcode == Instruction::Mul)
+ InterchangeableMask = MulBIT | ShlBIT;
+ else // Instruction::UDiv
+ InterchangeableMask = UDivBIT | LShrBIT;
+ }
break;
case Instruction::Add:
case Instruction::Sub:
diff --git a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
index 4e7a8270bde78..8fb52eedd9574 100644
--- a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
+++ b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
@@ -213,8 +213,7 @@ define void @lshr_div(ptr %p, ptr %s) {
; CHECK-LABEL: @lshr_div(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i16> [[TMP0]], <i16 3, i16 0, i16 0, i16 0>
-; CHECK-NEXT: [[TMP1:%.*]] = udiv <4 x i16> [[TMP2]], <i16 1, i16 5, i16 2, i16 3>
+; CHECK-NEXT: [[TMP1:%.*]] = udiv <4 x i16> [[TMP0]], <i16 8, i16 5, i16 2, i16 3>
; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
; CHECK-NEXT: ret void
;
@@ -248,8 +247,7 @@ define void @div_lshr(ptr %p, ptr %s) {
; CHECK-LABEL: @div_lshr(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP0]], <i16 5, i16 1, i16 2, i16 1>
-; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP2]], <i16 0, i16 3, i16 0, i16 9>
+; CHECK-NEXT: [[TMP1:%.*]] = udiv <4 x i16> [[TMP0]], <i16 5, i16 8, i16 2, i16 512>
; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
; CHECK-NEXT: ret void
;
@@ -283,8 +281,8 @@ define void @div_lshr_too_large(ptr %p, ptr %s) {
; CHECK-LABEL: @div_lshr_too_large(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = udiv <4 x i16> [[TMP0]], <i16 5, i16 1, i16 2, i16 1>
-; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i16> [[TMP1]], <i16 0, i16 17, i16 0, i16 9>
+; CHECK-NEXT: [[TMP1:%.*]] = udiv <4 x i16> [[TMP0]], <i16 5, i16 1, i16 1, i16 1>
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i16> [[TMP1]], <i16 0, i16 17, i16 1, i16 9>
; CHECK-NEXT: store <4 x i16> [[TMP2]], ptr [[S:%.*]], align 2
; CHECK-NEXT: ret void
;
@@ -315,30 +313,12 @@ entry:
}
define void @lshr_div2(ptr %p, ptr %s) {
-; X86-LABEL: @lshr_div2(
-; X86-NEXT: entry:
-; X86-NEXT: [[P1:%.*]] = getelementptr i16, ptr [[P:%.*]], i64 1
-; X86-NEXT: [[P3:%.*]] = getelementptr i16, ptr [[P]], i64 3
-; X86-NEXT: [[L0:%.*]] = load i16, ptr [[P]], align 2
-; X86-NEXT: [[L3:%.*]] = load i16, ptr [[P3]], align 2
-; X86-NEXT: [[DIV0:%.*]] = lshr i16 [[L0]], 3
-; X86-NEXT: [[DIV3:%.*]] = lshr i16 [[L3]], 5
-; X86-NEXT: [[S1:%.*]] = getelementptr i16, ptr [[S:%.*]], i64 1
-; X86-NEXT: [[S3:%.*]] = getelementptr i16, ptr [[S]], i64 3
-; X86-NEXT: [[TMP0:%.*]] = load <2 x i16>, ptr [[P1]], align 2
-; X86-NEXT: [[TMP1:%.*]] = udiv <2 x i16> [[TMP0]], <i16 8, i16 2>
-; X86-NEXT: store i16 [[DIV0]], ptr [[S]], align 2
-; X86-NEXT: store <2 x i16> [[TMP1]], ptr [[S1]], align 2
-; X86-NEXT: store i16 [[DIV3]], ptr [[S3]], align 2
-; X86-NEXT: ret void
-;
-; AARCH64-LABEL: @lshr_div2(
-; AARCH64-NEXT: entry:
-; AARCH64-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
-; AARCH64-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP0]], <i16 3, i16 0, i16 0, i16 5>
-; AARCH64-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP1]], <i16 1, i16 8, i16 2, i16 1>
-; AARCH64-NEXT: store <4 x i16> [[TMP2]], ptr [[S:%.*]], align 2
-; AARCH64-NEXT: ret void
+; CHECK-LABEL: @lshr_div2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP0]], <i16 3, i16 3, i16 1, i16 5>
+; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
+; CHECK-NEXT: ret void
;
entry:
%p1 = getelementptr i16, ptr %p, i64 1
@@ -370,8 +350,7 @@ define void @div2_lshr(ptr %p, ptr %s) {
; CHECK-LABEL: @div2_lshr(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P:%.*]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP0]], <i16 8, i16 1, i16 4, i16 1>
-; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP2]], <i16 0, i16 3, i16 0, i16 4>
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i16> [[TMP0]], <i16 3, i16 3, i16 2, i16 4>
; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[S:%.*]], align 2
; CHECK-NEXT: ret void
;
>From 3d7bbf94128ead8b44d6f0680f9a14562fe07d8d Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 17 Feb 2026 11:08:52 -0800
Subject: [PATCH 05/15] [SLP] Split LShr/UDiv from Shl/Mul logic for safety
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 24 +++++++++++++++++--
1 file changed, 22 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c02a3bd7d3f10..620a061f821f1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1096,8 +1096,18 @@ class BinOpSameOpcodeHelper {
Constant *RHS;
switch (FromOpcode) {
case Instruction::Shl:
+ if (ToOpcode == Instruction::Mul) {
+ RHS = ConstantInt::get(
+ RHSType, APInt::getOneBitSet(FromCIValueBitWidth,
+ FromCIValue.getZExtValue()));
+ } else {
+ assert(FromCIValue.isZero() && "Cannot convert the instruction.");
+ RHS = ConstantExpr::getBinOpIdentity(ToOpcode, RHSType,
+ /*AllowRHSConstant=*/true);
+ }
+ break;
case Instruction::LShr:
- if (ToOpcode == Instruction::Mul || ToOpcode == Instruction::UDiv) {
+ if (ToOpcode == Instruction::UDiv) {
RHS = ConstantInt::get(
RHSType, APInt::getOneBitSet(FromCIValueBitWidth,
FromCIValue.getZExtValue()));
@@ -1108,9 +1118,19 @@ class BinOpSameOpcodeHelper {
}
break;
case Instruction::Mul:
+ assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
+ if (ToOpcode == Instruction::Shl) {
+ RHS = ConstantInt::get(
+ RHSType, APInt(FromCIValueBitWidth, FromCIValue.logBase2()));
+ } else {
+ assert(FromCIValue.isOne() && "Cannot convert the instruction.");
+ RHS = ConstantExpr::getBinOpIdentity(ToOpcode, RHSType,
+ /*AllowRHSConstant=*/true);
+ }
+ break;
case Instruction::UDiv:
assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
- if (ToOpcode == Instruction::Shl || ToOpcode == Instruction::LShr) {
+ if (ToOpcode == Instruction::LShr) {
RHS = ConstantInt::get(
RHSType, APInt(FromCIValueBitWidth, FromCIValue.logBase2()));
} else {
>From 69d50a584c94685aa366fb6ca6add6df7d9e3220 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 17 Feb 2026 12:28:51 -0800
Subject: [PATCH 06/15] [SLP] Fixup partial_vec_invalid_cost to restore
original intent
Produced invalid cost
---
.../RISCV/partial-vec-invalid-cost.ll | 22 +++++++++++++------
1 file changed, 15 insertions(+), 7 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
index 3c6ea5a7cce87..f5e6b0c48a207 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
@@ -7,18 +7,26 @@ define void @partial_vec_invalid_cost() #0 {
; CHECK-LABEL: define void @partial_vec_invalid_cost(
; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[OP_RDX3:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer)
+; CHECK-NEXT: [[LSHR_1:%.*]] = lshr i96 0, 1
+; CHECK-NEXT: [[LSHR_2:%.*]] = lshr i96 0, 0
+; CHECK-NEXT: [[ADD_1:%.*]] = add i96 0, 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i96> poison, i96 [[LSHR_1]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i96> [[TMP0]], i96 [[LSHR_2]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i96> [[TMP1]], i96 0, i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i96> [[TMP2]], i96 [[ADD_1]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i96> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT: [[OP_RDX3:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[RDX_OP]])
; CHECK-NEXT: [[STORE_THIS:%.*]] = zext i32 [[OP_RDX3]] to i96
; CHECK-NEXT: store i96 [[STORE_THIS]], ptr null, align 16
; CHECK-NEXT: ret void
;
entry:
- ; Test is broken, I don't think there are any pairs of binary ops that
- ; can give an invalid cost
- %lshr.1 = lshr i96 0, 0
- %lshr.2 = lshr i96 0, 0
- %add.0 = add i96 0, 0
- %add.1 = add i96 0, 0
+
+ %lshr.1 = lshr i96 0, 1 ; These ops
+ %lshr.2 = lshr i96 0, 0 ; return an
+ %add.0 = add i96 0, 0 ; invalid
+ %add.1 = add i96 0, 1 ; vector cost.
%trunc.i96.1 = trunc i96 %lshr.1 to i32 ; These ops
%trunc.i96.2 = trunc i96 %lshr.2 to i32 ; return an
>From 0ae1b3ad1c14ffa44e6826a5b3ac4bdccf789150 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 17 Feb 2026 14:35:08 -0800
Subject: [PATCH 07/15] [SLP] Further split UDiv and Mul cases
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 620a061f821f1..f7f3fb43ebfcf 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1252,25 +1252,28 @@ class BinOpSameOpcodeHelper {
InterchangeableMask = CIValue.isZero() ? CanBeAll : UDivBIT | LShrBIT;
break;
case Instruction::Mul:
- case Instruction::UDiv:
if (CIValue.isOne()) {
InterchangeableMask = CanBeAll;
break;
}
- if (CIValue.isPowerOf2()) {
- if (Opcode == Instruction::Mul)
- InterchangeableMask = MulBIT | ShlBIT;
- else // Instruction::UDiv
- InterchangeableMask = UDivBIT | LShrBIT;
+ if (CIValue.isPowerOf2())
+ InterchangeableMask = MulBIT | ShlBIT;
+ break;
+ case Instruction::UDiv:
+ if (CIValue.isOne()) {
+ InterchangeableMask = CanBeAll;
+ break;
}
+ if (CIValue.isPowerOf2())
+ InterchangeableMask = UDivBIT | LShrBIT;
break;
case Instruction::Add:
case Instruction::Sub:
InterchangeableMask = CIValue.isZero() ? CanBeAll : SubBIT | AddBIT;
break;
case Instruction::And:
- if (CIValue.isAllOnes())
- InterchangeableMask = CanBeAll;
+ if (CIValue.isAllOnes())
+ InterchangeableMask = CanBeAll;
break;
case Instruction::Xor:
if (CIValue.isZero())
>From b9f2b4ab42116dc23e369c0061f334599c476c1d Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 17 Feb 2026 17:56:07 -0800
Subject: [PATCH 08/15] [SLP] Formatting
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f7f3fb43ebfcf..0ab0b1cea6c6f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1257,7 +1257,7 @@ class BinOpSameOpcodeHelper {
break;
}
if (CIValue.isPowerOf2())
- InterchangeableMask = MulBIT | ShlBIT;
+ InterchangeableMask = MulBIT | ShlBIT;
break;
case Instruction::UDiv:
if (CIValue.isOne()) {
@@ -1265,15 +1265,15 @@ class BinOpSameOpcodeHelper {
break;
}
if (CIValue.isPowerOf2())
- InterchangeableMask = UDivBIT | LShrBIT;
+ InterchangeableMask = UDivBIT | LShrBIT;
break;
case Instruction::Add:
case Instruction::Sub:
InterchangeableMask = CIValue.isZero() ? CanBeAll : SubBIT | AddBIT;
break;
case Instruction::And:
- if (CIValue.isAllOnes())
- InterchangeableMask = CanBeAll;
+ if (CIValue.isAllOnes())
+ InterchangeableMask = CanBeAll;
break;
case Instruction::Xor:
if (CIValue.isZero())
>From 4cc4f5f78dbbc45ecbf901154077904a8a49010d Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Mon, 16 Mar 2026 15:27:06 -0700
Subject: [PATCH 09/15] Remove extra newline
---
llvm/test/Transforms/SLPVectorizer/semanticly-same.ll | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
index 9d4249df96a82..c51d65b739b80 100644
--- a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
+++ b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
@@ -413,4 +413,3 @@ entry:
store i16 %add3, ptr %s3
ret void
}
-
>From 1e10c18c6085ed4e71fbaf7cf8e7cb7fb47648db Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Mon, 16 Mar 2026 15:28:43 -0700
Subject: [PATCH 10/15] Remove unneeded --check-prefixes
---
llvm/test/Transforms/SLPVectorizer/semanticly-same.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
index c51d65b739b80..db5fc8d3fbccd 100644
--- a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
+++ b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s --check-prefixes=CHECK %}
-; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s --check-prefixes=CHECK %}
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s}
; Don't care about the profitability with these tests, just want to demonstrate the ability
; to combine opcodes
>From f313c04e8e6b3467f70006c1cdada384cf198931 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Wed, 18 Mar 2026 13:04:27 -0700
Subject: [PATCH 11/15] Fix FileCheck call
---
llvm/test/Transforms/SLPVectorizer/semanticly-same.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
index db5fc8d3fbccd..ca3d5e6872d8c 100644
--- a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
+++ b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s}
-; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s}
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -slp-threshold=-50 < %s | FileCheck %s %}
; Don't care about the profitability with these tests, just want to demonstrate the ability
; to combine opcodes
>From 15d6606c5ea6ca43ff0933fb7ecb8bad33064650 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Wed, 18 Mar 2026 13:37:06 -0700
Subject: [PATCH 12/15] [SLP] Don't allow lshr %x, 0 to combine with arbitrary
binary operations
Shouldn't occur in real input (since is a NOP).
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 11 +++-----
.../X86/matching-insert-point-for-nodes.ll | 10 ++++---
.../SLPVectorizer/X86/no_alternate_divrem.ll | 19 +++++---------
.../X86/non-power-of-2-subvectors-insert.ll | 7 +++--
.../X86/reschedule-only-scheduled.ll | 26 +++++++++----------
5 files changed, 30 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0ab0b1cea6c6f..381c6dd39dbe8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1238,9 +1238,8 @@ class BinOpSameOpcodeHelper {
MaskType InterchangeableMask = OpcodeInMaskForm;
ConstantInt *CI = isBinOpWithConstantInt(I).first;
if (CI) {
- constexpr MaskType CanBeAll = XorBIT | OrBIT | AndBIT | SubBIT | AddBIT |
- MulBIT | AShrBIT | ShlBIT | LShrBIT |
- UDivBIT;
+ constexpr MaskType CanBeAll =
+ XorBIT | OrBIT | AndBIT | SubBIT | AddBIT | MulBIT | AShrBIT | ShlBIT;
const APInt &CIValue = CI->getValue();
switch (Opcode) {
case Instruction::Shl:
@@ -1249,7 +1248,7 @@ class BinOpSameOpcodeHelper {
break;
case Instruction::LShr:
if (CIValue.ult(CIValue.getBitWidth()))
- InterchangeableMask = CIValue.isZero() ? CanBeAll : UDivBIT | LShrBIT;
+ InterchangeableMask = UDivBIT | LShrBIT;
break;
case Instruction::Mul:
if (CIValue.isOne()) {
@@ -1260,10 +1259,6 @@ class BinOpSameOpcodeHelper {
InterchangeableMask = MulBIT | ShlBIT;
break;
case Instruction::UDiv:
- if (CIValue.isOne()) {
- InterchangeableMask = CanBeAll;
- break;
- }
if (CIValue.isPowerOf2())
InterchangeableMask = UDivBIT | LShrBIT;
break;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/matching-insert-point-for-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/matching-insert-point-for-nodes.ll
index 8a6705295df81..5e85ecd610ebd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/matching-insert-point-for-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/matching-insert-point-for-nodes.ll
@@ -6,8 +6,8 @@ define i32 @test() {
; CHECK-NEXT: [[BB:.*]]:
; CHECK-NEXT: br label %[[BB1:.*]]
; CHECK: [[BB1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ [[TMP14:%.*]], %[[BB24:.*]] ], [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i32> [ [[TMP13:%.*]], %[[BB24]] ], [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ [[TMP16:%.*]], %[[BB24:.*]] ], [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i32> [ [[TMP17:%.*]], %[[BB24]] ], [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB]] ]
; CHECK-NEXT: br i1 false, label %[[BB4:.*]], label %[[BB11:.*]]
; CHECK: [[BB4]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi <2 x double> [ zeroinitializer, %[[BB1]] ]
@@ -33,8 +33,10 @@ define i32 @test() {
; CHECK-NEXT: [[TMP9:%.*]] = lshr <4 x i32> [[TMP8]], <i32 poison, i32 poison, i32 0, i32 0>
; CHECK-NEXT: [[TMP10:%.*]] = and <4 x i32> [[TMP9]], <i32 poison, i32 poison, i32 0, i32 -1>
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
-; CHECK-NEXT: [[TMP14]] = lshr <4 x i32> [[TMP11]], [[TMP10]]
-; CHECK-NEXT: [[TMP13]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT: [[TMP14:%.*]] = lshr <4 x i32> [[TMP11]], [[TMP10]]
+; CHECK-NEXT: [[TMP15:%.*]] = or <4 x i32> [[TMP11]], [[TMP10]]
+; CHECK-NEXT: [[TMP16]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> [[TMP15]], <4 x i32> <i32 poison, i32 poison, i32 2, i32 7>
+; CHECK-NEXT: [[TMP17]] = shufflevector <4 x i32> [[TMP16]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, <4 x i32> <i32 poison, i32 poison, i32 2, i32 7>
; CHECK-NEXT: br label %[[BB1]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll b/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
index edb69f5ca8293..a888027479817 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
@@ -51,21 +51,14 @@ entry:
define void @test_add_udiv(ptr %arr1, ptr %arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
; CHECK-LABEL: @test_add_udiv(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 2
-; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr i32, ptr [[ARR1]], i32 3
-; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[GEP1_2]], align 4
-; CHECK-NEXT: [[V3:%.*]] = load i32, ptr [[GEP1_3]], align 4
-; CHECK-NEXT: [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[ARR1]], align 4
-; CHECK-NEXT: [[RES2:%.*]] = udiv i32 [[V2]], [[Y2]]
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[A0:%.*]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[ARR1:%.*]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[A2:%.*]], i32 2
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], <i32 1, i32 1, i32 42, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[A0:%.*]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A1:%.*]], i32 1
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[A3:%.*]], i32 3
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[RES2]], i32 2
-; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1146, i32 146, i32 0, i32 0>
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[V3]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> <i32 1146, i32 146, i32 0, i32 0>, [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = udiv <4 x i32> [[TMP0]], [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = add nsw <4 x i32> [[TMP8]], [[TMP4]]
; CHECK-NEXT: store <4 x i32> [[TMP9]], ptr [[ARR2:%.*]], align 4
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
index d85d5a61df53b..3e9bd781bfea1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
@@ -3,10 +3,9 @@
define void @test() {
; CHECK-LABEL: define void @test() {
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <14 x i64> <i64 undef, i64 undef, i64 0, i64 1, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 0, i64 undef>, <14 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 1, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>, <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <14 x i64> [[TMP1]], <14 x i64> <i64 0, i64 0, i64 0, i64 0, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>, <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 14, i32 15, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <14 x i64> [[TMP2]], <14 x i64> <i64 0, i64 0, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>, <14 x i32> <i32 14, i32 15, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <14 x i64> [[TMP3]], <14 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 3, i32 7, i32 8, i32 9, i32 3, i32 10, i32 11, i32 12, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i64> poison, i64 1, i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 1, i64 0, i64 undef, i64 undef, i64 undef, i64 undef>, <16 x i64> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i64> [[TMP6]], <16 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 3, i32 7, i32 8, i32 9, i32 3, i32 10, i32 11, i32 12, i32 3>
; CHECK-NEXT: [[TMP8:%.*]] = trunc <16 x i64> [[TMP7]] to <16 x i1>
; CHECK-NEXT: [[TMP9:%.*]] = or <16 x i1> [[TMP8]], zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = freeze <16 x i1> [[TMP9]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reschedule-only-scheduled.ll b/llvm/test/Transforms/SLPVectorizer/X86/reschedule-only-scheduled.ll
index 1301907b2e032..2dd6b395597c3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reschedule-only-scheduled.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reschedule-only-scheduled.ll
@@ -5,22 +5,20 @@ define i16 @test() {
; CHECK-LABEL: define i16 @test() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = lshr i32 0, 0
+; CHECK-NEXT: [[TMP10:%.*]] = shl i32 [[TMP0]], 0
; CHECK-NEXT: [[CALL99_I:%.*]] = call i32 @llvm.bswap.i32(i32 0)
+; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[CALL99_I]], 0
; CHECK-NEXT: [[CALL7_I45:%.*]] = tail call i32 null(i32 0)
-; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 0, 0
-; CHECK-NEXT: [[TMP2:%.*]] = shl i32 0, 0
-; CHECK-NEXT: [[UNSCLEAR78_I:%.*]] = and i32 [[TMP2]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <12 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 0, i32 0>, i32 [[TMP0]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <12 x i32> [[TMP3]], i32 [[CALL99_I]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <12 x i32> [[TMP4]], i32 [[CALL7_I45]], i32 8
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <12 x i32> [[TMP5]], i32 [[TMP1]], i32 9
-; CHECK-NEXT: [[TMP7:%.*]] = shl <12 x i32> [[TMP6]], zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = and <12 x i32> [[TMP7]], zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <12 x i32> [[TMP8]], <12 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 poison, i32 poison, i32 poison, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <32 x i32> [[TMP10]], i32 [[UNSCLEAR78_I]], i32 16
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <32 x i32> [[TMP11]], <32 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 32, i32 33, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <32 x i32> [[TMP16]], <32 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 6, i32 7, i32 7, i32 8, i32 9, i32 10, i32 10, i32 11, i32 12, i32 13, i32 13, i32 14, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[CALL7_I45]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <28 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 poison>, i32 [[TMP10]], i32 4
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <28 x i32> [[TMP4]], i32 [[TMP2]], i32 5
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <28 x i32> [[TMP5]], <28 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 28, i32 29, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <28 x i32> [[TMP6]], i32 [[TMP8]], i32 12
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <28 x i32> [[TMP7]], <28 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 poison>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <28 x i32> [[TMP16]], <28 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 28, i32 29, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <28 x i32> [[TMP9]], <28 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 28, i32 29, i32 30, i32 31, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27>
+; CHECK-NEXT: [[TMP11:%.*]] = and <28 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison>, [[TMP17]]
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <28 x i32> [[TMP11]], <28 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 6, i32 7, i32 7, i32 8, i32 9, i32 10, i32 10, i32 11, i32 12, i32 13, i32 13, i32 14, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
; CHECK-NEXT: [[TMP13:%.*]] = icmp ne <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1>, [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = bitcast <32 x i1> [[TMP13]] to i32
; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.ctpop.i32(i32 [[TMP14]])
>From 617db5cdc9ea64977f685192e3839ca380a6758e Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 8 Oct 2026 08:46:17 -0700
Subject: [PATCH 13/15] Lint
---
.../SLPVectorizer/SLPCompatibilityAnalysis.cpp | 12 ++++++------
.../SLPVectorizer/SLPCompatibilityAnalysis.h | 10 +++++-----
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
index 4feeff6c9bfa9..0a29dafe35734 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
@@ -181,9 +181,9 @@ SmallVector<Value *> BinOpSameOpcodeHelper::InterchangeableInfo::getOperand(
break;
case Instruction::LShr:
if (ToOpcode == Instruction::UDiv) {
- RHS = ConstantInt::get(
- RHSType, APInt::getOneBitSet(FromCIValueBitWidth,
- FromCIValue.getZExtValue()));
+ RHS = ConstantInt::get(RHSType,
+ APInt::getOneBitSet(FromCIValueBitWidth,
+ FromCIValue.getZExtValue()));
} else {
assert(FromCIValue.isZero() && "Cannot convert the instruction.");
RHS = ConstantExpr::getBinOpIdentity(ToOpcode, RHSType,
@@ -205,7 +205,7 @@ SmallVector<Value *> BinOpSameOpcodeHelper::InterchangeableInfo::getOperand(
assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
if (ToOpcode == Instruction::LShr) {
RHS = ConstantInt::get(
- RHSType, APInt(FromCIValueBitWidth, FromCIValue.logBase2()));
+ RHSType, APInt(FromCIValueBitWidth, FromCIValue.logBase2()));
} else {
assert(FromCIValue.isOne() && "Cannot convert the instruction.");
RHS = ConstantExpr::getBinOpIdentity(ToOpcode, RHSType,
@@ -265,8 +265,8 @@ bool BinOpSameOpcodeHelper::add(const Instruction *I) {
"BinOpSameOpcodeHelper only accepts BinaryOperator.");
unsigned Opcode = I->getOpcode();
MaskType OpcodeInMaskForm;
- // Prefer Shl, AShr, Mul, Add, Sub, And, Or, Xor, FAdd, FSub, LShr, and UDiv over
- // MainOp.
+ // Prefer Shl, AShr, Mul, Add, Sub, And, Or, Xor, FAdd, FSub, LShr, and UDiv
+ // over MainOp.
switch (Opcode) {
case Instruction::Shl:
OpcodeInMaskForm = ShlBIT;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
index ba453bb749131..2e28bcc96fb2c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
@@ -54,10 +54,9 @@ class BinOpSameOpcodeHelper {
using MaskType = std::uint_fast32_t;
/// Sort SupportedOp because it is used by binary_search.
constexpr static unsigned SupportedOp[] = {
- Instruction::Add, Instruction::FAdd, Instruction::Sub, Instruction::FSub,
- Instruction::Mul, Instruction::UDiv,
- Instruction::Shl, Instruction::LShr, Instruction::AShr, Instruction::And,
- Instruction::Or, Instruction::Xor};
+ Instruction::Add, Instruction::FAdd, Instruction::Sub, Instruction::FSub,
+ Instruction::Mul, Instruction::UDiv, Instruction::Shl, Instruction::LShr,
+ Instruction::AShr, Instruction::And, Instruction::Or, Instruction::Xor};
static_assert(llvm::is_sorted_constexpr(SupportedOp) &&
"SupportedOp is not sorted.");
enum : MaskType {
@@ -88,7 +87,8 @@ class BinOpSameOpcodeHelper {
const Instruction *I = nullptr;
/// The bit it sets represents whether MainOp can be converted to.
MaskType Mask = MainOpBIT | XorBIT | OrBIT | AndBIT | SubBIT | AddBIT |
- MulBIT | AShrBIT | ShlBIT | FSubBIT | FAddBIT | LShrBIT | UDivBIT;
+ MulBIT | AShrBIT | ShlBIT | FSubBIT | FAddBIT | LShrBIT |
+ UDivBIT;
/// We cannot create an interchangeable instruction that does not exist in
/// VL. For example, VL [x + 0, y * 1] can be converted to [x << 0, y << 0],
/// but << does not exist in VL. In the end, we convert VL to [x * 1, y *
>From 0c0356daf70218e8b5b62a1995084f482f06b5cf Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Sun, 11 Oct 2026 19:09:48 -0700
Subject: [PATCH 14/15] Remove dead code
---
.../SLPCompatibilityAnalysis.cpp | 26 +++++++------------
1 file changed, 9 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
index 0a29dafe35734..29dc9b107c6e9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
@@ -180,15 +180,11 @@ SmallVector<Value *> BinOpSameOpcodeHelper::InterchangeableInfo::getOperand(
}
break;
case Instruction::LShr:
- if (ToOpcode == Instruction::UDiv) {
- RHS = ConstantInt::get(RHSType,
- APInt::getOneBitSet(FromCIValueBitWidth,
- FromCIValue.getZExtValue()));
- } else {
- assert(FromCIValue.isZero() && "Cannot convert the instruction.");
- RHS = ConstantExpr::getBinOpIdentity(ToOpcode, RHSType,
- /*AllowRHSConstant=*/true);
- }
+ assert(ToOpcode == Instruction::UDiv &&
+ "Only support LShr to UDiv conversions.");
+ RHS = ConstantInt::get(
+ RHSType,
+ APInt::getOneBitSet(FromCIValueBitWidth, FromCIValue.getZExtValue()));
break;
case Instruction::Mul:
assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
@@ -203,14 +199,10 @@ SmallVector<Value *> BinOpSameOpcodeHelper::InterchangeableInfo::getOperand(
break;
case Instruction::UDiv:
assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
- if (ToOpcode == Instruction::LShr) {
- RHS = ConstantInt::get(
- RHSType, APInt(FromCIValueBitWidth, FromCIValue.logBase2()));
- } else {
- assert(FromCIValue.isOne() && "Cannot convert the instruction.");
- RHS = ConstantExpr::getBinOpIdentity(ToOpcode, RHSType,
- /*AllowRHSConstant=*/true);
- }
+ assert(ToOpcode == Instruction::LShr &&
+ "Only support UDiv to LShr conversion");
+ RHS = ConstantInt::get(
+ RHSType, APInt(FromCIValueBitWidth, FromCIValue.logBase2()));
break;
case Instruction::Add:
case Instruction::Sub:
>From b6ecdbed97c4d5c00282e44c0b932766e2c10fed Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Sun, 11 Oct 2026 19:09:57 -0700
Subject: [PATCH 15/15] Update comment
---
.../Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
index 2e28bcc96fb2c..486ab081d54be 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
@@ -79,7 +79,7 @@ class BinOpSameOpcodeHelper {
/// integer opcodes) or a ConstantFP (for FAdd/FSub).
/// The second return value represents the operand position. We check the
/// right-hand side first (1). If the right hand side is not a constant and
- /// the instruction is neither Sub, FSub, Shl, nor AShr, we then check the
+ /// the instruction is neither Sub, FSub, Shl, AShr, LShr, nor UDiv we then check the
/// left hand side (0).
static std::pair<Constant *, unsigned>
isBinOpWithConstant(const Instruction *I);
More information about the llvm-commits
mailing list