[llvm] [AMDGPU] Price the packed form of a vector of i1 (PR #217327)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 05:18:08 PDT 2026
https://github.com/michaelselehov updated https://github.com/llvm/llvm-project/pull/217327
>From d9742e90077fdd4e102139b1d12c3cdcf3544c41 Mon Sep 17 00:00:00 2001
From: Michael Selehov <michael.selehov at amd.com>
Date: Fri, 14 Aug 2026 09:35:39 -0500
Subject: [PATCH] [AMDGPU] Price the packed form of a vector of i1
A vector of i1 has no packed form on this target: every element lives in its own
mask. When an instruction needs the elements as the bits of one integer, the
target builds that integer with a select, a shift and an or per element. The
generic cost model charges about one instruction per element for that work, and
nothing at all for the reverse.
Instruction counts after codegen give 4.0 to 4.8 per element to pack the masks
and 2.6 to 2.9 to unpack them, from gfx900 to gfx1201. This patch charges 4 and
3. It prices the bitcast in both directions, and the add, the and and the or
reductions, which all go through the packed form. A xor reduction stays in the
masks, so it keeps its generic cost.
The add reduction is the one that fixes a regression in the rocSPARSE csrgemm
kernel, where SLP replaced a cheap scalar chain of adds with the packed form.
Assisted-by: Claude Opus
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 44 ++++
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 5 +
.../Analysis/CostModel/AMDGPU/bitcast-mask.ll | 81 ++++++++
.../CostModel/AMDGPU/reduce-add-i1.ll | 52 +++++
.../Analysis/CostModel/AMDGPU/reduce-and.ll | 32 +--
.../Analysis/CostModel/AMDGPU/reduce-or.ll | 32 +--
.../SLPVectorizer/AMDGPU/reduction-i1-mask.ll | 190 ++++++++++++++++++
7 files changed, 404 insertions(+), 32 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/AMDGPU/bitcast-mask.ll
create mode 100644 llvm/test/Analysis/CostModel/AMDGPU/reduce-add-i1.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction-i1-mask.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 980e26082064f..65b6a49983adc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -980,6 +980,41 @@ InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
return BaseT::getCFInstrCost(Opcode, CostKind, I);
}
+// A vector of i1 has no packed form on this target: every element lives in its
+// own mask. Measured instruction counts per element to pack the masks into an
+// integer, and to unpack them again.
+static constexpr unsigned MaskPackCostPerElt = 4;
+static constexpr unsigned MaskUnpackCostPerElt = 3;
+
+/// Returns the number of elements when \p Ty is a fixed vector of i1 with more
+/// than one element.
+static std::optional<unsigned> getPackedMaskElts(Type *Ty) {
+ auto *FVT = dyn_cast<FixedVectorType>(Ty);
+ if (FVT && FVT->getElementType()->isIntegerTy(1) && FVT->getNumElements() > 1)
+ return FVT->getNumElements();
+ return std::nullopt;
+}
+
+InstructionCost GCNTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
+ Type *Src,
+ TTI::CastContextHint CCH,
+ TTI::TargetCostKind CostKind,
+ const Instruction *I) const {
+ // A bitcast between a vector of i1 and an integer packs or unpacks a mask.
+ if (Opcode == Instruction::BitCast) {
+ if (std::optional<unsigned> Elts = getPackedMaskElts(Src);
+ Elts && Dst->isIntegerTy())
+ return InstructionCost(MaskPackCostPerElt) * *Elts *
+ getFullRateInstrCost();
+ if (std::optional<unsigned> Elts = getPackedMaskElts(Dst);
+ Elts && Src->isIntegerTy())
+ return InstructionCost(MaskUnpackCostPerElt) * *Elts *
+ getFullRateInstrCost();
+ }
+
+ return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+}
+
InstructionCost
GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
std::optional<FastMathFlags> FMF,
@@ -987,6 +1022,15 @@ GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
if (TTI::requiresOrderedReduction(FMF))
return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
+ // These three reductions over a vector of i1 go through the packed form of
+ // the mask, and the packing dominates their cost.
+ if (Opcode == Instruction::Add || Opcode == Instruction::And ||
+ Opcode == Instruction::Or) {
+ if (std::optional<unsigned> Elts = getPackedMaskElts(Ty))
+ return InstructionCost(MaskPackCostPerElt) * *Elts *
+ getFullRateInstrCost();
+ }
+
EVT OrigTy = TLI->getValueType(DL, Ty);
// Computes cost on targets that have packed math instructions(which support
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index df7b6d339e6c2..2be0a5ef329cb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -268,6 +268,11 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
return InlinerVectorBonusPercent;
}
+ InstructionCost
+ getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+ TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
+ const Instruction *I = nullptr) const override;
+
InstructionCost
getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
std::optional<FastMathFlags> FMF,
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/bitcast-mask.ll b/llvm/test/Analysis/CostModel/AMDGPU/bitcast-mask.ll
new file mode 100644
index 0000000000000..2498f1755e2ee
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/bitcast-mask.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -mtriple=amdgpu9.00-unknown-amdhsa -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck -check-prefixes=ALL %s
+; RUN: opt < %s -mtriple=amdgpu10.30-unknown-amdhsa -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck -check-prefixes=ALL %s
+; RUN: opt < %s -mtriple=amdgpu12.01-unknown-amdhsa -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck -check-prefixes=ALL %s
+; RUN: opt < %s -mtriple=amdgpu12.01-unknown-amdhsa -mattr=-real-true16 -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck -check-prefixes=ALL %s
+; RUN: opt < %s -mtriple=amdgpu12.01-unknown-amdhsa -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck -check-prefixes=ALL-SIZE %s
+; END.
+
+; A bitcast between a vector of i1 and an integer packs or unpacks a mask. The
+; cost is per element, and it does not depend on the subtarget.
+
+define void @pack() {
+; ALL-LABEL: 'pack'
+; ALL-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V2 = bitcast <2 x i1> poison to i2
+; ALL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V4 = bitcast <4 x i1> poison to i4
+; ALL-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V8 = bitcast <8 x i1> poison to i8
+; ALL-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V16 = bitcast <16 x i1> poison to i16
+; ALL-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %V32 = bitcast <32 x i1> poison to i32
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'pack'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V2 = bitcast <2 x i1> poison to i2
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V4 = bitcast <4 x i1> poison to i4
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V8 = bitcast <8 x i1> poison to i8
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V16 = bitcast <16 x i1> poison to i16
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %V32 = bitcast <32 x i1> poison to i32
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %V2 = bitcast <2 x i1> poison to i2
+ %V4 = bitcast <4 x i1> poison to i4
+ %V8 = bitcast <8 x i1> poison to i8
+ %V16 = bitcast <16 x i1> poison to i16
+ %V32 = bitcast <32 x i1> poison to i32
+ ret void
+}
+
+define void @unpack() {
+; ALL-LABEL: 'unpack'
+; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = bitcast i2 poison to <2 x i1>
+; ALL-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4 = bitcast i4 poison to <4 x i1>
+; ALL-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8 = bitcast i8 poison to <8 x i1>
+; ALL-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16 = bitcast i16 poison to <16 x i1>
+; ALL-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %V32 = bitcast i32 poison to <32 x i1>
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'unpack'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = bitcast i2 poison to <2 x i1>
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4 = bitcast i4 poison to <4 x i1>
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8 = bitcast i8 poison to <8 x i1>
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16 = bitcast i16 poison to <16 x i1>
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %V32 = bitcast i32 poison to <32 x i1>
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %V2 = bitcast i2 poison to <2 x i1>
+ %V4 = bitcast i4 poison to <4 x i1>
+ %V8 = bitcast i8 poison to <8 x i1>
+ %V16 = bitcast i16 poison to <16 x i1>
+ %V32 = bitcast i32 poison to <32 x i1>
+ ret void
+}
+
+; A single element needs no packing, a bitcast between two vectors of i1 is a
+; move, and a scalable vector keeps the generic cost.
+define void @unchanged() {
+; ALL-LABEL: 'unchanged'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1 = bitcast <1 x i1> poison to i1
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %W1 = bitcast i1 poison to <1 x i1>
+; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %S = bitcast <vscale x 8 x i1> poison to <vscale x 8 x i1>
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'unchanged'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1 = bitcast <1 x i1> poison to i1
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %W1 = bitcast i1 poison to <1 x i1>
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %S = bitcast <vscale x 8 x i1> poison to <vscale x 8 x i1>
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %V1 = bitcast <1 x i1> poison to i1
+ %W1 = bitcast i1 poison to <1 x i1>
+ %S = bitcast <vscale x 8 x i1> poison to <vscale x 8 x i1>
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-add-i1.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-add-i1.ll
new file mode 100644
index 0000000000000..f438526a70961
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-add-i1.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -mtriple=amdgpu9.00-unknown-amdhsa -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck -check-prefixes=ALL %s
+; RUN: opt < %s -mtriple=amdgpu10.30-unknown-amdhsa -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck -check-prefixes=ALL %s
+; RUN: opt < %s -mtriple=amdgpu12.01-unknown-amdhsa -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck -check-prefixes=ALL %s
+; RUN: opt < %s -mtriple=amdgpu12.01-unknown-amdhsa -mattr=-real-true16 -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck -check-prefixes=ALL %s
+; RUN: opt < %s -mtriple=amdgpu12.01-unknown-amdhsa -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck -check-prefixes=ALL-SIZE %s
+; END.
+
+; The cost is the packed form of the mask, which is what SLP emits. It takes 4
+; instructions per element on every generation from GFX9 to GFX12.
+
+define void @reduce_add_i1() {
+; ALL-LABEL: 'reduce_add_i1'
+; ALL-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i1 @llvm.vector.reduce.add.v2i1(<2 x i1> poison)
+; ALL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> poison)
+; ALL-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i1 @llvm.vector.reduce.add.v8i1(<8 x i1> poison)
+; ALL-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i1 @llvm.vector.reduce.add.v16i1(<16 x i1> poison)
+; ALL-NEXT: Cost Model: Found an estimated cost of 256 for instruction: %V64 = call i1 @llvm.vector.reduce.add.v64i1(<64 x i1> poison)
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'reduce_add_i1'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i1 @llvm.vector.reduce.add.v2i1(<2 x i1> poison)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> poison)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i1 @llvm.vector.reduce.add.v8i1(<8 x i1> poison)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i1 @llvm.vector.reduce.add.v16i1(<16 x i1> poison)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 256 for instruction: %V64 = call i1 @llvm.vector.reduce.add.v64i1(<64 x i1> poison)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %V2 = call i1 @llvm.vector.reduce.add.v2i1(<2 x i1> poison)
+ %V4 = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> poison)
+ %V8 = call i1 @llvm.vector.reduce.add.v8i1(<8 x i1> poison)
+ %V16 = call i1 @llvm.vector.reduce.add.v16i1(<16 x i1> poison)
+ %V64 = call i1 @llvm.vector.reduce.add.v64i1(<64 x i1> poison)
+ ret void
+}
+
+; One element needs no packing, and a scalable vector keeps the generic cost.
+define void @unchanged(<vscale x 8 x i1> %s) {
+; ALL-LABEL: 'unchanged'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.add.v1i1(<1 x i1> poison)
+; ALL-NEXT: Cost Model: Invalid cost for instruction: %S = call i1 @llvm.vector.reduce.add.nxv8i1(<vscale x 8 x i1> %s)
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'unchanged'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.add.v1i1(<1 x i1> poison)
+; ALL-SIZE-NEXT: Cost Model: Invalid cost for instruction: %S = call i1 @llvm.vector.reduce.add.nxv8i1(<vscale x 8 x i1> %s)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %V1 = call i1 @llvm.vector.reduce.add.v1i1(<1 x i1> poison)
+ %S = call i1 @llvm.vector.reduce.add.nxv8i1(<vscale x 8 x i1> %s)
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
index 45971bd6f4ba0..daeb790e77c9d 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
@@ -6,26 +6,26 @@
define i32 @reduce_i1(i32 %arg) {
; ALL-LABEL: 'reduce_i1'
; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 260 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 256 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 512 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 1024 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
;
; ALL-SIZE-LABEL: 'reduce_i1'
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 260 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 256 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 512 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1024 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
;
%V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
index e92285c25410a..ae5f08e7283ba 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
@@ -6,26 +6,26 @@
define i32 @reduce_i1(i32 %arg) {
; ALL-LABEL: 'reduce_i1'
; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
-; ALL-NEXT: Cost Model: Found an estimated cost of 260 for instruction: %V256 = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 256 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 512 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; ALL-NEXT: Cost Model: Found an estimated cost of 1024 for instruction: %V256 = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> undef)
; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
;
; ALL-SIZE-LABEL: 'reduce_i1'
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 260 for instruction: %V256 = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 256 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 512 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1024 for instruction: %V256 = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
;
%V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction-i1-mask.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction-i1-mask.ll
new file mode 100644
index 0000000000000..a23e972aefbac
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction-i1-mask.ll
@@ -0,0 +1,190 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu12.01-amd-amdhsa < %s | FileCheck %s
+; A low threshold makes SLP emit the reduction, so the cost is what keeps it away.
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu12.01-amd-amdhsa \
+; RUN: -slp-threshold=-100 < %s | FileCheck %s --check-prefix=FORCED
+
+; A reduction over a vector of i1 goes through the packed form of the mask, and
+; the packing costs more than the scalar chain that SLP starts from.
+
+; This counts how many entries of a table are smaller than a key.
+define i32 @count_smaller(ptr addrspace(3) %tab, i32 %key) {
+; CHECK-LABEL: define i32 @count_smaller(
+; CHECK-SAME: ptr addrspace(3) [[TAB:%.*]], i32 [[KEY:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 2
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 3
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 4
+; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 5
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 6
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 7
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr addrspace(3) [[TAB]], align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr addrspace(3) [[P1]], align 4
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr addrspace(3) [[P2]], align 4
+; CHECK-NEXT: [[V3:%.*]] = load i32, ptr addrspace(3) [[P3]], align 4
+; CHECK-NEXT: [[V4:%.*]] = load i32, ptr addrspace(3) [[P4]], align 4
+; CHECK-NEXT: [[V5:%.*]] = load i32, ptr addrspace(3) [[P5]], align 4
+; CHECK-NEXT: [[V6:%.*]] = load i32, ptr addrspace(3) [[P6]], align 4
+; CHECK-NEXT: [[V7:%.*]] = load i32, ptr addrspace(3) [[P7]], align 4
+; CHECK-NEXT: [[C0:%.*]] = icmp slt i32 [[V0]], [[KEY]]
+; CHECK-NEXT: [[C1:%.*]] = icmp slt i32 [[V1]], [[KEY]]
+; CHECK-NEXT: [[C2:%.*]] = icmp slt i32 [[V2]], [[KEY]]
+; CHECK-NEXT: [[C3:%.*]] = icmp slt i32 [[V3]], [[KEY]]
+; CHECK-NEXT: [[C4:%.*]] = icmp slt i32 [[V4]], [[KEY]]
+; CHECK-NEXT: [[C5:%.*]] = icmp slt i32 [[V5]], [[KEY]]
+; CHECK-NEXT: [[C6:%.*]] = icmp slt i32 [[V6]], [[KEY]]
+; CHECK-NEXT: [[C7:%.*]] = icmp slt i32 [[V7]], [[KEY]]
+; CHECK-NEXT: [[Z0:%.*]] = zext i1 [[C0]] to i32
+; CHECK-NEXT: [[Z1:%.*]] = zext i1 [[C1]] to i32
+; CHECK-NEXT: [[Z2:%.*]] = zext i1 [[C2]] to i32
+; CHECK-NEXT: [[Z3:%.*]] = zext i1 [[C3]] to i32
+; CHECK-NEXT: [[Z4:%.*]] = zext i1 [[C4]] to i32
+; CHECK-NEXT: [[Z5:%.*]] = zext i1 [[C5]] to i32
+; CHECK-NEXT: [[Z6:%.*]] = zext i1 [[C6]] to i32
+; CHECK-NEXT: [[Z7:%.*]] = zext i1 [[C7]] to i32
+; CHECK-NEXT: [[S1:%.*]] = add i32 [[Z0]], [[Z1]]
+; CHECK-NEXT: [[S2:%.*]] = add i32 [[S1]], [[Z2]]
+; CHECK-NEXT: [[S3:%.*]] = add i32 [[S2]], [[Z3]]
+; CHECK-NEXT: [[S4:%.*]] = add i32 [[S3]], [[Z4]]
+; CHECK-NEXT: [[S5:%.*]] = add i32 [[S4]], [[Z5]]
+; CHECK-NEXT: [[S6:%.*]] = add i32 [[S5]], [[Z6]]
+; CHECK-NEXT: [[S7:%.*]] = add i32 [[S6]], [[Z7]]
+; CHECK-NEXT: ret i32 [[S7]]
+;
+; FORCED-LABEL: define i32 @count_smaller(
+; FORCED-SAME: ptr addrspace(3) [[TAB:%.*]], i32 [[KEY:%.*]]) {
+; FORCED-NEXT: [[ENTRY:.*:]]
+; FORCED-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr addrspace(3) [[TAB]], align 4
+; FORCED-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[KEY]], i64 0
+; FORCED-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
+; FORCED-NEXT: [[TMP3:%.*]] = icmp slt <8 x i32> [[TMP0]], [[TMP2]]
+; FORCED-NEXT: [[TMP4:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; FORCED-NEXT: [[TMP5:%.*]] = call i8 @llvm.ctpop.i8(i8 [[TMP4]])
+; FORCED-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; FORCED-NEXT: ret i32 [[TMP6]]
+;
+entry:
+ %p1 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 1
+ %p2 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 2
+ %p3 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 3
+ %p4 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 4
+ %p5 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 5
+ %p6 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 6
+ %p7 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 7
+ %v0 = load i32, ptr addrspace(3) %tab, align 4
+ %v1 = load i32, ptr addrspace(3) %p1, align 4
+ %v2 = load i32, ptr addrspace(3) %p2, align 4
+ %v3 = load i32, ptr addrspace(3) %p3, align 4
+ %v4 = load i32, ptr addrspace(3) %p4, align 4
+ %v5 = load i32, ptr addrspace(3) %p5, align 4
+ %v6 = load i32, ptr addrspace(3) %p6, align 4
+ %v7 = load i32, ptr addrspace(3) %p7, align 4
+ %c0 = icmp slt i32 %v0, %key
+ %c1 = icmp slt i32 %v1, %key
+ %c2 = icmp slt i32 %v2, %key
+ %c3 = icmp slt i32 %v3, %key
+ %c4 = icmp slt i32 %v4, %key
+ %c5 = icmp slt i32 %v5, %key
+ %c6 = icmp slt i32 %v6, %key
+ %c7 = icmp slt i32 %v7, %key
+ %z0 = zext i1 %c0 to i32
+ %z1 = zext i1 %c1 to i32
+ %z2 = zext i1 %c2 to i32
+ %z3 = zext i1 %c3 to i32
+ %z4 = zext i1 %c4 to i32
+ %z5 = zext i1 %c5 to i32
+ %z6 = zext i1 %c6 to i32
+ %z7 = zext i1 %c7 to i32
+ %s1 = add i32 %z0, %z1
+ %s2 = add i32 %s1, %z2
+ %s3 = add i32 %s2, %z3
+ %s4 = add i32 %s3, %z4
+ %s5 = add i32 %s4, %z5
+ %s6 = add i32 %s5, %z6
+ %s7 = add i32 %s6, %z7
+ ret i32 %s7
+}
+
+; An and reduction over a vector of i1 packs the mask as well, and then compares
+; it against all ones. This checks that every entry of the table is smaller.
+define i1 @all_smaller(ptr addrspace(3) %tab, i32 %key) {
+; CHECK-LABEL: define i1 @all_smaller(
+; CHECK-SAME: ptr addrspace(3) [[TAB:%.*]], i32 [[KEY:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 2
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 3
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 4
+; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 5
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 6
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TAB]], i32 7
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr addrspace(3) [[TAB]], align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr addrspace(3) [[P1]], align 4
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr addrspace(3) [[P2]], align 4
+; CHECK-NEXT: [[V3:%.*]] = load i32, ptr addrspace(3) [[P3]], align 4
+; CHECK-NEXT: [[V4:%.*]] = load i32, ptr addrspace(3) [[P4]], align 4
+; CHECK-NEXT: [[V5:%.*]] = load i32, ptr addrspace(3) [[P5]], align 4
+; CHECK-NEXT: [[V6:%.*]] = load i32, ptr addrspace(3) [[P6]], align 4
+; CHECK-NEXT: [[V7:%.*]] = load i32, ptr addrspace(3) [[P7]], align 4
+; CHECK-NEXT: [[C0:%.*]] = icmp slt i32 [[V0]], [[KEY]]
+; CHECK-NEXT: [[C1:%.*]] = icmp slt i32 [[V1]], [[KEY]]
+; CHECK-NEXT: [[C2:%.*]] = icmp slt i32 [[V2]], [[KEY]]
+; CHECK-NEXT: [[C3:%.*]] = icmp slt i32 [[V3]], [[KEY]]
+; CHECK-NEXT: [[C4:%.*]] = icmp slt i32 [[V4]], [[KEY]]
+; CHECK-NEXT: [[C5:%.*]] = icmp slt i32 [[V5]], [[KEY]]
+; CHECK-NEXT: [[C6:%.*]] = icmp slt i32 [[V6]], [[KEY]]
+; CHECK-NEXT: [[C7:%.*]] = icmp slt i32 [[V7]], [[KEY]]
+; CHECK-NEXT: [[A1:%.*]] = and i1 [[C0]], [[C1]]
+; CHECK-NEXT: [[A2:%.*]] = and i1 [[A1]], [[C2]]
+; CHECK-NEXT: [[A3:%.*]] = and i1 [[A2]], [[C3]]
+; CHECK-NEXT: [[A4:%.*]] = and i1 [[A3]], [[C4]]
+; CHECK-NEXT: [[A5:%.*]] = and i1 [[A4]], [[C5]]
+; CHECK-NEXT: [[A6:%.*]] = and i1 [[A5]], [[C6]]
+; CHECK-NEXT: [[A7:%.*]] = and i1 [[A6]], [[C7]]
+; CHECK-NEXT: ret i1 [[A7]]
+;
+; FORCED-LABEL: define i1 @all_smaller(
+; FORCED-SAME: ptr addrspace(3) [[TAB:%.*]], i32 [[KEY:%.*]]) {
+; FORCED-NEXT: [[ENTRY:.*:]]
+; FORCED-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr addrspace(3) [[TAB]], align 4
+; FORCED-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[KEY]], i64 0
+; FORCED-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
+; FORCED-NEXT: [[TMP3:%.*]] = icmp slt <8 x i32> [[TMP0]], [[TMP2]]
+; FORCED-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP3]])
+; FORCED-NEXT: ret i1 [[TMP4]]
+;
+entry:
+ %p1 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 1
+ %p2 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 2
+ %p3 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 3
+ %p4 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 4
+ %p5 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 5
+ %p6 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 6
+ %p7 = getelementptr inbounds i32, ptr addrspace(3) %tab, i32 7
+ %v0 = load i32, ptr addrspace(3) %tab, align 4
+ %v1 = load i32, ptr addrspace(3) %p1, align 4
+ %v2 = load i32, ptr addrspace(3) %p2, align 4
+ %v3 = load i32, ptr addrspace(3) %p3, align 4
+ %v4 = load i32, ptr addrspace(3) %p4, align 4
+ %v5 = load i32, ptr addrspace(3) %p5, align 4
+ %v6 = load i32, ptr addrspace(3) %p6, align 4
+ %v7 = load i32, ptr addrspace(3) %p7, align 4
+ %c0 = icmp slt i32 %v0, %key
+ %c1 = icmp slt i32 %v1, %key
+ %c2 = icmp slt i32 %v2, %key
+ %c3 = icmp slt i32 %v3, %key
+ %c4 = icmp slt i32 %v4, %key
+ %c5 = icmp slt i32 %v5, %key
+ %c6 = icmp slt i32 %v6, %key
+ %c7 = icmp slt i32 %v7, %key
+ %a1 = and i1 %c0, %c1
+ %a2 = and i1 %a1, %c2
+ %a3 = and i1 %a2, %c3
+ %a4 = and i1 %a3, %c4
+ %a5 = and i1 %a4, %c5
+ %a6 = and i1 %a5, %c6
+ %a7 = and i1 %a6, %c7
+ ret i1 %a7
+}
More information about the llvm-commits
mailing list