[llvm] 15a3238 - [AArch64] Lower extends of boolean vector loads via scalar load (#203394)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 22 00:06:52 PDT 2026
Author: Weiwen He
Date: 2026-06-22T08:06:46+01:00
New Revision: 15a3238566e799e3dd0e473672307712be8528e6
URL: https://github.com/llvm/llvm-project/commit/15a3238566e799e3dd0e473672307712be8528e6
DIFF: https://github.com/llvm/llvm-project/commit/15a3238566e799e3dd0e473672307712be8528e6.diff
LOG: [AArch64] Lower extends of boolean vector loads via scalar load (#203394)
Replace a `load <N x i1>` under a sext/zext with a scalar load +
bitcast, so the `combineToExtendBoolVectorInReg` helper can apply,
avoiding scalarization.
Optimisation for the SVE case with a predicate load to be added in a
follow up.
Fixes #200325
Added:
llvm/test/CodeGen/AArch64/extend-bool-vector-load.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 67ef911117eff..01bc2495563c0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25021,6 +25021,50 @@ static SDValue performExtendDuplaneTruncCombine(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(NewDupOpc, SDLoc(N), DstVT, Src, Dup.getOperand(1));
}
+static SDValue
+performExtendToBoolVectorLoadCombine(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const AArch64Subtarget &Subtarget) {
+ EVT VT = N->getValueType(0);
+ SDValue N0 = N->getOperand(0);
+ auto *LN0 = dyn_cast<LoadSDNode>(N0);
+ // Match an extend of a normal load from a boolean vector (<N x i1>) to a
+ // fixed-length integer vector.
+ if (!LN0 || !ISD::isNormalLoad(LN0) ||
+ N0.getValueType().getScalarType() != MVT::i1 || !VT.isFixedLengthVector())
+ return SDValue();
+
+ // Only fold a load with a single use that is simple (not volatile or atomic),
+ // so it is safe to replace with a scalar load of the same bytes.
+ if (!N0.hasOneUse() || !LN0->isSimple())
+ return SDValue();
+
+ SDLoc DL(N);
+ unsigned NumElts = VT.getVectorNumElements();
+ unsigned EltSizeInBits = VT.getScalarSizeInBits();
+ // Load <N x i1> as a scalar iN, then bitcast it back to <N x i1> so the
+ // generic combineToExtendBoolVectorInReg helper can apply. That helper
+ // requires the scalar to be broadcast across the result elements, so only
+ // proceed when that precondition holds.
+ // TODO: Use a predicate load for SVE vectors.
+ bool CanSplatOrSplit =
+ NumElts <= EltSizeInBits || NumElts % EltSizeInBits == 0;
+ if (Subtarget.isNeonAvailable() && CanSplatOrSplit) {
+ EVT ScalarVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
+ SDValue ScalarLd = DAG.getLoad(ScalarVT, DL, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getMemOperand());
+ SDValue Bitcast = DAG.getBitcast(LN0->getValueType(0), ScalarLd);
+ if (SDValue V = combineToExtendBoolVectorInReg(
+ N->getOpcode(), DL, VT, Bitcast, DAG, DCI, Subtarget)) {
+ // Redirect the old load's chain users to the new scalar load.
+ DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ScalarLd.getValue(1));
+ return V;
+ }
+ }
+
+ return SDValue();
+}
+
static SDValue performExtendCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
@@ -25054,6 +25098,9 @@ static SDValue performExtendCombine(SDNode *N,
DAG, DCI, *Subtarget))
return V;
+ if (SDValue V = performExtendToBoolVectorLoadCombine(N, DAG, DCI, *Subtarget))
+ return V;
+
if (N->getValueType(0).isFixedLengthVector() &&
N->getOpcode() == ISD::SIGN_EXTEND &&
N->getOperand(0)->getOpcode() == ISD::SETCC)
diff --git a/llvm/test/CodeGen/AArch64/extend-bool-vector-load.ll b/llvm/test/CodeGen/AArch64/extend-bool-vector-load.ll
new file mode 100644
index 0000000000000..1dcded38365f8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/extend-bool-vector-load.ll
@@ -0,0 +1,294 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+define <16 x i8> @sext_v16i1_v16i8(ptr %p) {
+; CHECK-LE-LABEL: sext_v16i1_v16i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI0_0
+; CHECK-LE-NEXT: ldr h1, [x0]
+; CHECK-LE-NEXT: ldr q0, [x8, :lo12:.LCPI0_0]
+; CHECK-LE-NEXT: adrp x8, .LCPI0_1
+; CHECK-LE-NEXT: tbl v0.16b, { v1.16b }, v0.16b
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI0_1]
+; CHECK-LE-NEXT: cmtst v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: sext_v16i1_v16i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldr h0, [x0]
+; CHECK-BE-NEXT: adrp x8, .LCPI0_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: adrp x8, .LCPI0_1
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_1
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: cmtst v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %l = load <16 x i1>, ptr %p, align 4
+ %e = sext <16 x i1> %l to <16 x i8>
+ ret <16 x i8> %e
+}
+
+define <16 x i8> @zext_v16i1_v16i8(ptr %p) {
+; CHECK-LE-LABEL: zext_v16i1_v16i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI1_0
+; CHECK-LE-NEXT: ldr h1, [x0]
+; CHECK-LE-NEXT: ldr q0, [x8, :lo12:.LCPI1_0]
+; CHECK-LE-NEXT: adrp x8, .LCPI1_1
+; CHECK-LE-NEXT: tbl v0.16b, { v1.16b }, v0.16b
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI1_1]
+; CHECK-LE-NEXT: cmtst v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: ushr v0.16b, v0.16b, #7
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: zext_v16i1_v16i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldr h0, [x0]
+; CHECK-BE-NEXT: adrp x8, .LCPI1_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI1_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: adrp x8, .LCPI1_1
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI1_1
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: cmtst v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: ushr v0.16b, v0.16b, #7
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %l = load <16 x i1>, ptr %p, align 4
+ %e = zext <16 x i1> %l to <16 x i8>
+ ret <16 x i8> %e
+}
+
+define <32 x i8> @sext_v32i1_v32i8(ptr %p) {
+; CHECK-LE-LABEL: sext_v32i1_v32i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI2_0
+; CHECK-LE-NEXT: adrp x9, .LCPI2_2
+; CHECK-LE-NEXT: ldr s1, [x0]
+; CHECK-LE-NEXT: ldr q0, [x8, :lo12:.LCPI2_0]
+; CHECK-LE-NEXT: ldr q2, [x9, :lo12:.LCPI2_2]
+; CHECK-LE-NEXT: adrp x8, .LCPI2_1
+; CHECK-LE-NEXT: tbl v0.16b, { v1.16b }, v0.16b
+; CHECK-LE-NEXT: tbl v1.16b, { v1.16b }, v2.16b
+; CHECK-LE-NEXT: ldr q2, [x8, :lo12:.LCPI2_1]
+; CHECK-LE-NEXT: cmtst v0.16b, v0.16b, v2.16b
+; CHECK-LE-NEXT: cmtst v1.16b, v1.16b, v2.16b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: sext_v32i1_v32i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldr s0, [x0]
+; CHECK-BE-NEXT: adrp x8, .LCPI2_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: adrp x8, .LCPI2_2
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI2_2
+; CHECK-BE-NEXT: rev32 v0.16b, v0.16b
+; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-NEXT: adrp x8, .LCPI2_1
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI2_1
+; CHECK-BE-NEXT: tbl v1.16b, { v0.16b }, v1.16b
+; CHECK-BE-NEXT: tbl v0.16b, { v0.16b }, v2.16b
+; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-NEXT: cmtst v1.16b, v1.16b, v2.16b
+; CHECK-BE-NEXT: cmtst v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %l = load <32 x i1>, ptr %p, align 4
+ %e = sext <32 x i1> %l to <32 x i8>
+ ret <32 x i8> %e
+}
+
+define <4 x i32> @sext_v4i1_v4i32(ptr %p) {
+; CHECK-LE-LABEL: sext_v4i1_v4i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldr b0, [x0]
+; CHECK-LE-NEXT: adrp x8, .LCPI3_0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-LE-NEXT: dup v0.4s, v0.s[0]
+; CHECK-LE-NEXT: cmtst v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: sext_v4i1_v4i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldr b0, [x0]
+; CHECK-BE-NEXT: adrp x8, .LCPI3_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI3_0
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: dup v0.4s, v0.s[0]
+; CHECK-BE-NEXT: cmtst v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %l = load <4 x i1>, ptr %p, align 4
+ %e = sext <4 x i1> %l to <4 x i32>
+ ret <4 x i32> %e
+}
+
+define <16 x i8> @sext_v16i1_volatile(ptr %p) {
+; CHECK-LE-LABEL: sext_v16i1_volatile:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldrh w8, [x0]
+; CHECK-LE-NEXT: and w10, w8, #0x1
+; CHECK-LE-NEXT: ubfx w9, w8, #1, #1
+; CHECK-LE-NEXT: fmov s0, w10
+; CHECK-LE-NEXT: mov v0.b[1], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #2, #1
+; CHECK-LE-NEXT: mov v0.b[2], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #3, #1
+; CHECK-LE-NEXT: mov v0.b[3], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #4, #1
+; CHECK-LE-NEXT: mov v0.b[4], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #5, #1
+; CHECK-LE-NEXT: mov v0.b[5], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #6, #1
+; CHECK-LE-NEXT: mov v0.b[6], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #7, #1
+; CHECK-LE-NEXT: mov v0.b[7], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #8, #1
+; CHECK-LE-NEXT: mov v0.b[8], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #9, #1
+; CHECK-LE-NEXT: mov v0.b[9], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #10, #1
+; CHECK-LE-NEXT: mov v0.b[10], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #11, #1
+; CHECK-LE-NEXT: mov v0.b[11], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #12, #1
+; CHECK-LE-NEXT: mov v0.b[12], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #13, #1
+; CHECK-LE-NEXT: mov v0.b[13], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #14, #1
+; CHECK-LE-NEXT: lsr w8, w8, #15
+; CHECK-LE-NEXT: mov v0.b[14], w9
+; CHECK-LE-NEXT: mov v0.b[15], w8
+; CHECK-LE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: sext_v16i1_volatile:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldrh w8, [x0]
+; CHECK-BE-NEXT: lsr w9, w8, #15
+; CHECK-BE-NEXT: ubfx w10, w8, #14, #1
+; CHECK-BE-NEXT: fmov s0, w9
+; CHECK-BE-NEXT: ubfx w9, w8, #13, #1
+; CHECK-BE-NEXT: mov v0.b[1], w10
+; CHECK-BE-NEXT: mov v0.b[2], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #12, #1
+; CHECK-BE-NEXT: mov v0.b[3], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #11, #1
+; CHECK-BE-NEXT: mov v0.b[4], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #10, #1
+; CHECK-BE-NEXT: mov v0.b[5], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #9, #1
+; CHECK-BE-NEXT: mov v0.b[6], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #8, #1
+; CHECK-BE-NEXT: mov v0.b[7], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #7, #1
+; CHECK-BE-NEXT: mov v0.b[8], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #6, #1
+; CHECK-BE-NEXT: mov v0.b[9], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #5, #1
+; CHECK-BE-NEXT: mov v0.b[10], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #4, #1
+; CHECK-BE-NEXT: mov v0.b[11], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #3, #1
+; CHECK-BE-NEXT: mov v0.b[12], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #2, #1
+; CHECK-BE-NEXT: mov v0.b[13], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #1, #1
+; CHECK-BE-NEXT: and w8, w8, #0x1
+; CHECK-BE-NEXT: mov v0.b[14], w9
+; CHECK-BE-NEXT: mov v0.b[15], w8
+; CHECK-BE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %l = load volatile <16 x i1>, ptr %p, align 4
+ %e = sext <16 x i1> %l to <16 x i8>
+ ret <16 x i8> %e
+}
+
+define <12 x i8> @sext_v12i1_v12i8(ptr %p) {
+; CHECK-LE-LABEL: sext_v12i1_v12i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldrh w8, [x0]
+; CHECK-LE-NEXT: and w10, w8, #0x1
+; CHECK-LE-NEXT: ubfx w9, w8, #1, #1
+; CHECK-LE-NEXT: fmov s0, w10
+; CHECK-LE-NEXT: mov v0.b[1], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #2, #1
+; CHECK-LE-NEXT: mov v0.b[2], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #3, #1
+; CHECK-LE-NEXT: mov v0.b[3], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #4, #1
+; CHECK-LE-NEXT: mov v0.b[4], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #5, #1
+; CHECK-LE-NEXT: mov v0.b[5], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #6, #1
+; CHECK-LE-NEXT: mov v0.b[6], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #7, #1
+; CHECK-LE-NEXT: mov v0.b[7], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #8, #1
+; CHECK-LE-NEXT: mov v0.b[8], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #9, #1
+; CHECK-LE-NEXT: mov v0.b[9], w9
+; CHECK-LE-NEXT: ubfx w9, w8, #10, #1
+; CHECK-LE-NEXT: lsr w8, w8, #11
+; CHECK-LE-NEXT: mov v0.b[10], w9
+; CHECK-LE-NEXT: mov v0.b[11], w8
+; CHECK-LE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: sext_v12i1_v12i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldrh w8, [x0]
+; CHECK-BE-NEXT: lsr w9, w8, #11
+; CHECK-BE-NEXT: ubfx w10, w8, #10, #1
+; CHECK-BE-NEXT: fmov s0, w9
+; CHECK-BE-NEXT: ubfx w9, w8, #9, #1
+; CHECK-BE-NEXT: mov v0.b[1], w10
+; CHECK-BE-NEXT: mov v0.b[2], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #8, #1
+; CHECK-BE-NEXT: mov v0.b[3], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #7, #1
+; CHECK-BE-NEXT: mov v0.b[4], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #6, #1
+; CHECK-BE-NEXT: mov v0.b[5], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #5, #1
+; CHECK-BE-NEXT: mov v0.b[6], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #4, #1
+; CHECK-BE-NEXT: mov v0.b[7], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #3, #1
+; CHECK-BE-NEXT: mov v0.b[8], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #2, #1
+; CHECK-BE-NEXT: mov v0.b[9], w9
+; CHECK-BE-NEXT: ubfx w9, w8, #1, #1
+; CHECK-BE-NEXT: and w8, w8, #0x1
+; CHECK-BE-NEXT: mov v0.b[10], w9
+; CHECK-BE-NEXT: mov v0.b[11], w8
+; CHECK-BE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %l = load <12 x i1>, ptr %p, align 4
+ %e = sext <12 x i1> %l to <12 x i8>
+ ret <12 x i8> %e
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
More information about the llvm-commits
mailing list