[llvm] [X86] Fold splat AND on VGF2P8AFFINEQB source (PR #193364)
via llvm-commits
llvm-commits at lists.llvm.org
Wed May 13 05:47:58 PDT 2026
https://github.com/WalterKruger updated https://github.com/llvm/llvm-project/pull/193364
>From 46adb09f48d7fa0665a2f306af4366678d69ff1c Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Wed, 22 Apr 2026 10:04:02 +1000
Subject: [PATCH 1/9] [NFC] Baseline test without GFNI fold
Regression check line test without the fold, demonstrating the current code gen.
---
.../test/CodeGen/X86/gfni-operand-and-fold.ll | 169 ++++++++++++++++++
1 file changed, 169 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
diff --git a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
new file mode 100644
index 0000000000000..bbac9237e84a8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+gfni,+avx2 | FileCheck %s --check-prefixes=AVX
+
+declare <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
+
+define <16 x i8> @test_const_splat_on_const_matrix(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_const_splat_on_const_matrix:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: retq
+ %and = and <16 x i8> %src, splat(i8 15)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 0)
+ ret <16 x i8> %gfni
+}
+
+define <32 x i8> @test_const_splat_on_const_matrix256(<32 x i8> %src) nounwind {
+; AVX-LABEL: test_const_splat_on_const_matrix256:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX-NEXT: retq
+ %and = and <32 x i8> %src, splat(i8 15)
+ %gfni = call <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8> %and, <32 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 0)
+ ret <32 x i8> %gfni
+}
+
+define <16 x i8> @test_const_splat_parity_matrix(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_const_splat_parity_matrix:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: retq
+ %and = and <16 x i8> %src, splat(i8 85)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 1, i8 3, i8 7, i8 15, i8 31, i8 63, i8 127, i8 255, i8 1, i8 3, i8 7, i8 15, i8 31, i8 63, i8 127, i8 255>, i8 0)
+ ret <16 x i8> %gfni
+}
+
+define <16 x i8> @test_const_splat_on_const_matrix_nonzero_imm(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_const_splat_on_const_matrix_nonzero_imm:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: retq
+ %and = and <16 x i8> %src, splat(i8 15)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 127)
+ ret <16 x i8> %gfni
+}
+
+define <16 x i8> @test_var_splat_on_const_matrix(<16 x i8> %src, i8 %scalar) nounwind {
+; AVX-LABEL: test_var_splat_on_const_matrix:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd %edi, %xmm1
+; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: retq
+ %inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
+ %varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
+ %and = and <16 x i8> %src, %varsplat
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> splat(i8 2), i8 0)
+ ret <16 x i8> %gfni
+}
+
+define <16 x i8> @test_var_splat_parity_fill_matrix(<16 x i8> %src, i8 %scalar) nounwind {
+; AVX-LABEL: test_var_splat_parity_fill_matrix:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd %edi, %xmm1
+; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
+ %inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
+ %varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
+ %and = and <16 x i8> %src, %varsplat
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> splat(i8 -1), i8 0)
+ ret <16 x i8> %gfni
+}
+
+define <16 x i8> @test_var_splat_on_const_matrix_nonzero_imm(<16 x i8> %src, i8 %scalar) nounwind {
+; AVX-LABEL: test_var_splat_on_const_matrix_nonzero_imm:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd %edi, %xmm1
+; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $170, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: retq
+ %inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
+ %varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
+ %and = and <16 x i8> %src, %varsplat
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> splat(i8 2), i8 170)
+ ret <16 x i8> %gfni
+}
+
+;; Negative test: multi use should not fold
+define <16 x i8> @test_const_splat_on_const_matrix_multi_use(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_const_splat_on_const_matrix_multi_use:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX-NEXT: vpxor %xmm0, %xmm1, %xmm0
+; AVX-NEXT: retq
+ %matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
+ %and = and <16 x i8> %src, splat(i8 15)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ %xor = xor <16 x i8> %gfni, %and
+ ret <16 x i8> %xor
+}
+
+;; Negative test: don't fold as it may increase dependency chain
+define <16 x i8> @test_const_splat_on_var_operands(<16 x i8> %src, <16 x i8> %matrix) nounwind {
+; AVX-LABEL: test_const_splat_on_var_operands:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
+ %and = and <16 x i8> %src, splat(i8 1)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ ret <16 x i8> %gfni
+}
+
+;; Negative test: don't fold as it may increase dependency chain
+define <16 x i8> @test_var_splat_on_var_operands(<16 x i8> %src, <16 x i8> %matrix, i8 %scalar) nounwind {
+; AVX-LABEL: test_var_splat_on_var_operands:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd %edi, %xmm2
+; AVX-NEXT: vpbroadcastb %xmm2, %xmm2
+; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
+ %inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
+ %varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
+ %and = and <16 x i8> %src, %varsplat
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ ret <16 x i8> %gfni
+}
+
+;; Negative test: larger splats should not fold
+define <16 x i8> @test_const_splat16_on_const_matrix(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_const_splat16_on_const_matrix:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: retq
+ %splat = bitcast <8 x i16> splat(i16 1) to <16 x i8>
+ %and = and <16 x i8> %src, %splat
+ %matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ ret <16 x i8> %gfni
+}
+
+;; Negative test: larger splats should not fold
+define <16 x i8> @test_var_splat16_on_const_matrix(<16 x i8> %src, i16 %scalar) nounwind {
+; AVX-LABEL: test_var_splat16_on_const_matrix:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd %edi, %xmm1
+; AVX-NEXT: vpbroadcastw %xmm1, %xmm1
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: retq
+ %inlo = insertelement <8 x i16> poison, i16 %scalar, i64 0
+ %varsplat16 = shufflevector <8 x i16> %inlo, <8 x i16> poison, <8 x i32> zeroinitializer
+ %varsplat = bitcast <8 x i16> %varsplat16 to <16 x i8>
+ %and = and <16 x i8> %src, %varsplat
+ %matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ ret <16 x i8> %gfni
+}
>From 5bc3fab9de1b93ead0806c3ce714f0abe3e918c3 Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Wed, 22 Apr 2026 10:12:15 +1000
Subject: [PATCH 2/9] Fold AND on GF2P8AFFINEQB + test update
- Added the AND on the GF2P8AFFINEQB source fold.
- Updated the regression test to match new codegen.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 57 +++++++++++++++++++
.../test/CodeGen/X86/gfni-operand-and-fold.ll | 14 ++---
2 files changed, 61 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index aa5b864df5936..79528fce0723a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -62138,6 +62138,62 @@ static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+static SDValue combineAndOnGF2P8AFFINEQBOperand(SDNode *N, const SDLoc &DL,
+ SelectionDAG &DAG, EVT VT) {
+ using namespace SDPatternMatch;
+
+ SDValue X, Y, SplatOp;
+ APInt Imm, SplatVal, ConstUndef;
+ SmallVector<APInt> ConstEltBits;
+
+ // Removing the same bit within each matrix's row effectively treats the
+ // corresponding source bit like it is set to zero
+ // TODO: Add reverse fold when X is constant
+ if (sd_match(N, m_TernaryOp(X86ISD::GF2P8AFFINEQB,
+ m_OneUse(m_And(m_Value(X), m_Value(SplatOp))),
+ m_Value(Y), m_ConstInt(Imm))) &&
+ getTargetConstantBitsFromNode(Y, Y.getScalarValueSizeInBits(), ConstUndef,
+ ConstEltBits, /*AllowWholeUndefs=*/false)) {
+ bool splatIsConst =
+ X86::isConstantSplat(SplatOp, SplatVal, /*AllowPartialUndefs=*/false);
+
+ if (!(splatIsConst || DAG.isSplatValue(SplatOp, /*AllowUndefs=*/false) ||
+ SplatOp.getOpcode() == X86ISD::VBROADCAST ||
+ SplatOp.getOpcode() == X86ISD::VBROADCAST_LOAD) ||
+ SplatOp.getScalarValueSizeInBits() != 8)
+ return SDValue();
+
+ // ANDs with constants are not folded away this far into lowering
+ SDValue NewMatrix;
+ if (!splatIsConst) {
+ NewMatrix = DAG.getNode(ISD::AND, DL, VT, SplatOp, Y);
+ } else {
+ SmallVector<SDValue> FoldedAnd;
+ for (APInt &Elt : ConstEltBits)
+ FoldedAnd.push_back(DAG.getConstant(SplatVal & Elt, DL, MVT::i8));
+
+ NewMatrix = DAG.getBuildVector(VT, DL, FoldedAnd);
+ }
+
+ return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, X, NewMatrix,
+ DAG.getTargetConstant(Imm, DL, MVT::i8));
+ }
+
+ return SDValue();
+}
+
+static SDValue combineGF2P8AFFINEQB(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ EVT VT = N->getValueType(0);
+ // const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ SDLoc dl(N);
+
+ if (SDValue R = combineAndOnGF2P8AFFINEQBOperand(N, dl, DAG, VT))
+ return R;
+
+ return SDValue();
+}
+
// Optimize (fp16_to_fp (fp_to_fp16 X)) to VCVTPS2PH followed by VCVTPH2PS.
// Done as a combine because the lowering for fp16_to_fp and fp_to_fp16 produce
// extra instructions between the conversion due to going to scalar and back.
@@ -62737,6 +62793,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::VPMADD52H: return combineVPMADD52LH(N, DAG, DCI);
case X86ISD::KSHIFTL:
case X86ISD::KSHIFTR: return combineKSHIFT(N, DAG, DCI);
+ case X86ISD::GF2P8AFFINEQB: return combineGF2P8AFFINEQB(N, DAG, DCI);
case ISD::FP16_TO_FP: return combineFP16_TO_FP(N, DAG, Subtarget);
case ISD::STRICT_FP_EXTEND:
case ISD::FP_EXTEND: return combineFP_EXTEND(N, DAG, DCI, Subtarget);
diff --git a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
index bbac9237e84a8..80deda5901bfd 100644
--- a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
@@ -6,7 +6,6 @@ declare <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
define <16 x i8> @test_const_splat_on_const_matrix(<16 x i8> %src) nounwind {
; AVX-LABEL: test_const_splat_on_const_matrix:
; AVX: # %bb.0:
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX-NEXT: retq
%and = and <16 x i8> %src, splat(i8 15)
@@ -17,7 +16,6 @@ define <16 x i8> @test_const_splat_on_const_matrix(<16 x i8> %src) nounwind {
define <32 x i8> @test_const_splat_on_const_matrix256(<32 x i8> %src) nounwind {
; AVX-LABEL: test_const_splat_on_const_matrix256:
; AVX: # %bb.0:
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX-NEXT: retq
%and = and <32 x i8> %src, splat(i8 15)
@@ -28,7 +26,6 @@ define <32 x i8> @test_const_splat_on_const_matrix256(<32 x i8> %src) nounwind {
define <16 x i8> @test_const_splat_parity_matrix(<16 x i8> %src) nounwind {
; AVX-LABEL: test_const_splat_parity_matrix:
; AVX: # %bb.0:
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX-NEXT: retq
%and = and <16 x i8> %src, splat(i8 85)
@@ -39,7 +36,6 @@ define <16 x i8> @test_const_splat_parity_matrix(<16 x i8> %src) nounwind {
define <16 x i8> @test_const_splat_on_const_matrix_nonzero_imm(<16 x i8> %src) nounwind {
; AVX-LABEL: test_const_splat_on_const_matrix_nonzero_imm:
; AVX: # %bb.0:
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX-NEXT: vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX-NEXT: retq
%and = and <16 x i8> %src, splat(i8 15)
@@ -52,8 +48,8 @@ define <16 x i8> @test_var_splat_on_const_matrix(<16 x i8> %src, i8 %scalar) nou
; AVX: # %bb.0:
; AVX-NEXT: vmovd %edi, %xmm1
; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
-; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
%varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
@@ -67,8 +63,6 @@ define <16 x i8> @test_var_splat_parity_fill_matrix(<16 x i8> %src, i8 %scalar)
; AVX: # %bb.0:
; AVX-NEXT: vmovd %edi, %xmm1
; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
-; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
@@ -83,8 +77,8 @@ define <16 x i8> @test_var_splat_on_const_matrix_nonzero_imm(<16 x i8> %src, i8
; AVX: # %bb.0:
; AVX-NEXT: vmovd %edi, %xmm1
; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
-; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vgf2p8affineqb $170, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX-NEXT: vgf2p8affineqb $170, %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
%varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
>From 6550ce4e77daa4b7cfaa076716650065c7fa45f3 Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Thu, 23 Apr 2026 00:09:21 +1000
Subject: [PATCH 3/9] More aggressive multi-use + check-prefixes
- Made fold emission more aggressive by still allowing multi-use AND when splat is constant. Still beneficial as it shortens the dependency chain.
- Changes all check prefixes to `CHECK`.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 10 +-
.../test/CodeGen/X86/gfni-operand-and-fold.ll | 155 ++++++++++--------
2 files changed, 94 insertions(+), 71 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 79528fce0723a..8c393b884228d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -62149,14 +62149,18 @@ static SDValue combineAndOnGF2P8AFFINEQBOperand(SDNode *N, const SDLoc &DL,
// Removing the same bit within each matrix's row effectively treats the
// corresponding source bit like it is set to zero
// TODO: Add reverse fold when X is constant
- if (sd_match(N, m_TernaryOp(X86ISD::GF2P8AFFINEQB,
- m_OneUse(m_And(m_Value(X), m_Value(SplatOp))),
- m_Value(Y), m_ConstInt(Imm))) &&
+ if (sd_match(N, m_TernaryOp(X86ISD::GF2P8AFFINEQB, m_Value(AndOp), m_Value(Y),
+ m_ConstInt(Imm))) &&
+ sd_match(AndOp, m_And(m_Value(X), m_Value(SplatOp))) &&
getTargetConstantBitsFromNode(Y, Y.getScalarValueSizeInBits(), ConstUndef,
ConstEltBits, /*AllowWholeUndefs=*/false)) {
bool splatIsConst =
X86::isConstantSplat(SplatOp, SplatVal, /*AllowPartialUndefs=*/false);
+ // Can still shorten the chain when constant folded with the matrix
+ if (!AndOp->hasOneUse() && !splatIsConst)
+ return SDValue();
+
if (!(splatIsConst || DAG.isSplatValue(SplatOp, /*AllowUndefs=*/false) ||
SplatOp.getOpcode() == X86ISD::VBROADCAST ||
SplatOp.getOpcode() == X86ISD::VBROADCAST_LOAD) ||
diff --git a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
index 80deda5901bfd..c179bb7541336 100644
--- a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
@@ -1,56 +1,56 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+gfni,+avx2 | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+gfni,+avx2 | FileCheck %s
declare <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
define <16 x i8> @test_const_splat_on_const_matrix(<16 x i8> %src) nounwind {
-; AVX-LABEL: test_const_splat_on_const_matrix:
-; AVX: # %bb.0:
-; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_const_splat_on_const_matrix:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 15)
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 0)
ret <16 x i8> %gfni
}
define <32 x i8> @test_const_splat_on_const_matrix256(<32 x i8> %src) nounwind {
-; AVX-LABEL: test_const_splat_on_const_matrix256:
-; AVX: # %bb.0:
-; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_const_splat_on_const_matrix256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT: retq
%and = and <32 x i8> %src, splat(i8 15)
%gfni = call <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8> %and, <32 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 0)
ret <32 x i8> %gfni
}
define <16 x i8> @test_const_splat_parity_matrix(<16 x i8> %src) nounwind {
-; AVX-LABEL: test_const_splat_parity_matrix:
-; AVX: # %bb.0:
-; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_const_splat_parity_matrix:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 85)
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 1, i8 3, i8 7, i8 15, i8 31, i8 63, i8 127, i8 255, i8 1, i8 3, i8 7, i8 15, i8 31, i8 63, i8 127, i8 255>, i8 0)
ret <16 x i8> %gfni
}
define <16 x i8> @test_const_splat_on_const_matrix_nonzero_imm(<16 x i8> %src) nounwind {
-; AVX-LABEL: test_const_splat_on_const_matrix_nonzero_imm:
-; AVX: # %bb.0:
-; AVX-NEXT: vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_const_splat_on_const_matrix_nonzero_imm:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 15)
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 127)
ret <16 x i8> %gfni
}
define <16 x i8> @test_var_splat_on_const_matrix(<16 x i8> %src, i8 %scalar) nounwind {
-; AVX-LABEL: test_var_splat_on_const_matrix:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm1
-; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_var_splat_on_const_matrix:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovd %edi, %xmm1
+; CHECK-NEXT: vpbroadcastb %xmm1, %xmm1
+; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; CHECK-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
%varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
%and = and <16 x i8> %src, %varsplat
@@ -59,12 +59,12 @@ define <16 x i8> @test_var_splat_on_const_matrix(<16 x i8> %src, i8 %scalar) nou
}
define <16 x i8> @test_var_splat_parity_fill_matrix(<16 x i8> %src, i8 %scalar) nounwind {
-; AVX-LABEL: test_var_splat_parity_fill_matrix:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm1
-; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
-; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_var_splat_parity_fill_matrix:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovd %edi, %xmm1
+; CHECK-NEXT: vpbroadcastb %xmm1, %xmm1
+; CHECK-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
%varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
%and = and <16 x i8> %src, %varsplat
@@ -73,13 +73,13 @@ define <16 x i8> @test_var_splat_parity_fill_matrix(<16 x i8> %src, i8 %scalar)
}
define <16 x i8> @test_var_splat_on_const_matrix_nonzero_imm(<16 x i8> %src, i8 %scalar) nounwind {
-; AVX-LABEL: test_var_splat_on_const_matrix_nonzero_imm:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm1
-; AVX-NEXT: vpbroadcastb %xmm1, %xmm1
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX-NEXT: vgf2p8affineqb $170, %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_var_splat_on_const_matrix_nonzero_imm:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovd %edi, %xmm1
+; CHECK-NEXT: vpbroadcastb %xmm1, %xmm1
+; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; CHECK-NEXT: vgf2p8affineqb $170, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
%varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
%and = and <16 x i8> %src, %varsplat
@@ -87,14 +87,14 @@ define <16 x i8> @test_var_splat_on_const_matrix_nonzero_imm(<16 x i8> %src, i8
ret <16 x i8> %gfni
}
-;; Negative test: multi use should not fold
+;; Multi use still shortens dependency chain
define <16 x i8> @test_const_splat_on_const_matrix_multi_use(<16 x i8> %src) nounwind {
-; AVX-LABEL: test_const_splat_on_const_matrix_multi_use:
-; AVX: # %bb.0:
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX-NEXT: vpxor %xmm0, %xmm1, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_const_splat_on_const_matrix_multi_use:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: retq
%matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
%and = and <16 x i8> %src, splat(i8 15)
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
@@ -102,13 +102,32 @@ define <16 x i8> @test_const_splat_on_const_matrix_multi_use(<16 x i8> %src) nou
ret <16 x i8> %xor
}
+;; Negative test: multi use would generate an additional AND
+define <16 x i8> @test_var_splat_on_const_matrix_multi_use(<16 x i8> %src, i8 %scalar) nounwind {
+; CHECK-LABEL: test_var_splat_on_const_matrix_multi_use:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovd %edi, %xmm1
+; CHECK-NEXT: vpbroadcastb %xmm1, %xmm1
+; CHECK-NEXT: vpand %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; CHECK-NEXT: vpxor %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: retq
+ %inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
+ %varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
+ %matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
+ %and = and <16 x i8> %src, %varsplat
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ %xor = xor <16 x i8> %gfni, %and
+ ret <16 x i8> %xor
+}
+
;; Negative test: don't fold as it may increase dependency chain
define <16 x i8> @test_const_splat_on_var_operands(<16 x i8> %src, <16 x i8> %matrix) nounwind {
-; AVX-LABEL: test_const_splat_on_var_operands:
-; AVX: # %bb.0:
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_const_splat_on_var_operands:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 1)
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
ret <16 x i8> %gfni
@@ -116,13 +135,13 @@ define <16 x i8> @test_const_splat_on_var_operands(<16 x i8> %src, <16 x i8> %ma
;; Negative test: don't fold as it may increase dependency chain
define <16 x i8> @test_var_splat_on_var_operands(<16 x i8> %src, <16 x i8> %matrix, i8 %scalar) nounwind {
-; AVX-LABEL: test_var_splat_on_var_operands:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm2
-; AVX-NEXT: vpbroadcastb %xmm2, %xmm2
-; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_var_splat_on_var_operands:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovd %edi, %xmm2
+; CHECK-NEXT: vpbroadcastb %xmm2, %xmm2
+; CHECK-NEXT: vpand %xmm2, %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
%varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
%and = and <16 x i8> %src, %varsplat
@@ -132,11 +151,11 @@ define <16 x i8> @test_var_splat_on_var_operands(<16 x i8> %src, <16 x i8> %matr
;; Negative test: larger splats should not fold
define <16 x i8> @test_const_splat16_on_const_matrix(<16 x i8> %src) nounwind {
-; AVX-LABEL: test_const_splat16_on_const_matrix:
-; AVX: # %bb.0:
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_const_splat16_on_const_matrix:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: retq
%splat = bitcast <8 x i16> splat(i16 1) to <16 x i8>
%and = and <16 x i8> %src, %splat
%matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
@@ -146,13 +165,13 @@ define <16 x i8> @test_const_splat16_on_const_matrix(<16 x i8> %src) nounwind {
;; Negative test: larger splats should not fold
define <16 x i8> @test_var_splat16_on_const_matrix(<16 x i8> %src, i16 %scalar) nounwind {
-; AVX-LABEL: test_var_splat16_on_const_matrix:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm1
-; AVX-NEXT: vpbroadcastw %xmm1, %xmm1
-; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: retq
+; CHECK-LABEL: test_var_splat16_on_const_matrix:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovd %edi, %xmm1
+; CHECK-NEXT: vpbroadcastw %xmm1, %xmm1
+; CHECK-NEXT: vpand %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: retq
%inlo = insertelement <8 x i16> poison, i16 %scalar, i64 0
%varsplat16 = shufflevector <8 x i16> %inlo, <8 x i16> poison, <8 x i32> zeroinitializer
%varsplat = bitcast <8 x i16> %varsplat16 to <16 x i8>
>From ecf5daf7aa52183567d230c9928716a90b6dfe32 Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Thu, 23 Apr 2026 12:38:23 +1000
Subject: [PATCH 4/9] Remove opcode compares when detecting splats
The target specific opcode checks where added due to issues with variable splat detection. Apparently that was due to an unrelated issue that has since been resolved and so they are redundant.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 8c393b884228d..58d77e3b57d38 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -62161,9 +62161,7 @@ static SDValue combineAndOnGF2P8AFFINEQBOperand(SDNode *N, const SDLoc &DL,
if (!AndOp->hasOneUse() && !splatIsConst)
return SDValue();
- if (!(splatIsConst || DAG.isSplatValue(SplatOp, /*AllowUndefs=*/false) ||
- SplatOp.getOpcode() == X86ISD::VBROADCAST ||
- SplatOp.getOpcode() == X86ISD::VBROADCAST_LOAD) ||
+ if (!(splatIsConst || DAG.isSplatValue(SplatOp, /*AllowUndefs=*/false)) ||
SplatOp.getScalarValueSizeInBits() != 8)
return SDValue();
>From 297f9ff0964400418fe714452cf656b70579c920 Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Thu, 23 Apr 2026 13:10:25 +1000
Subject: [PATCH 5/9] Improve test clarity
- Change the name of `test_const_splat_parity_matrix` to better reflect why it exists and added a explanatory comment.
- Replace bit casted `splat(i64 ...)` to i8 build vectors. This was causing confusion on what needed to be splatted.
---
llvm/test/CodeGen/X86/gfni-operand-and-fold.ll | 17 +++++++----------
1 file changed, 7 insertions(+), 10 deletions(-)
diff --git a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
index c179bb7541336..011136eb4e51a 100644
--- a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
@@ -23,8 +23,9 @@ define <32 x i8> @test_const_splat_on_const_matrix256(<32 x i8> %src) nounwind {
ret <32 x i8> %gfni
}
-define <16 x i8> @test_const_splat_parity_matrix(<16 x i8> %src) nounwind {
-; CHECK-LABEL: test_const_splat_parity_matrix:
+;; Same situation as above but with a diffrent matrix
+define <16 x i8> @test_const_splat_alternative_const_matrix(<16 x i8> %src) nounwind {
+; CHECK-LABEL: test_const_splat_alternative_const_matrix:
; CHECK: # %bb.0:
; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; CHECK-NEXT: retq
@@ -95,9 +96,8 @@ define <16 x i8> @test_const_splat_on_const_matrix_multi_use(<16 x i8> %src) nou
; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; CHECK-NEXT: vpxor %xmm1, %xmm0, %xmm0
; CHECK-NEXT: retq
- %matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
%and = and <16 x i8> %src, splat(i8 15)
- %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128>, i8 0)
%xor = xor <16 x i8> %gfni, %and
ret <16 x i8> %xor
}
@@ -114,9 +114,8 @@ define <16 x i8> @test_var_splat_on_const_matrix_multi_use(<16 x i8> %src, i8 %s
; CHECK-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
%varsplat = shufflevector <16 x i8> %inlo, <16 x i8> poison, <16 x i32> zeroinitializer
- %matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
%and = and <16 x i8> %src, %varsplat
- %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128>, i8 0)
%xor = xor <16 x i8> %gfni, %and
ret <16 x i8> %xor
}
@@ -158,8 +157,7 @@ define <16 x i8> @test_const_splat16_on_const_matrix(<16 x i8> %src) nounwind {
; CHECK-NEXT: retq
%splat = bitcast <8 x i16> splat(i16 1) to <16 x i8>
%and = and <16 x i8> %src, %splat
- %matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
- %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128>, i8 0)
ret <16 x i8> %gfni
}
@@ -176,7 +174,6 @@ define <16 x i8> @test_var_splat16_on_const_matrix(<16 x i8> %src, i16 %scalar)
%varsplat16 = shufflevector <8 x i16> %inlo, <8 x i16> poison, <8 x i32> zeroinitializer
%varsplat = bitcast <8 x i16> %varsplat16 to <16 x i8>
%and = and <16 x i8> %src, %varsplat
- %matrix = bitcast <2 x i64> splat(i64 9223655728169885760) to <16 x i8>
- %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> %matrix, i8 0)
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128>, i8 0)
ret <16 x i8> %gfni
}
>From 6df38ae4455d39f5a1e45c16e9805753e0343e03 Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Thu, 23 Apr 2026 22:08:28 +1000
Subject: [PATCH 6/9] Fix missing AndOp & Pascal case
- The SDValue AndOp variable was missing.
- Changed `splatIsConst` to Pascal case to better match styling
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 15 +++++++--------
1 file changed, 7 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 58d77e3b57d38..9131cb97bdeb8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -62142,7 +62142,7 @@ static SDValue combineAndOnGF2P8AFFINEQBOperand(SDNode *N, const SDLoc &DL,
SelectionDAG &DAG, EVT VT) {
using namespace SDPatternMatch;
- SDValue X, Y, SplatOp;
+ SDValue X, Y, AndOp, SplatOp;
APInt Imm, SplatVal, ConstUndef;
SmallVector<APInt> ConstEltBits;
@@ -62154,20 +62154,20 @@ static SDValue combineAndOnGF2P8AFFINEQBOperand(SDNode *N, const SDLoc &DL,
sd_match(AndOp, m_And(m_Value(X), m_Value(SplatOp))) &&
getTargetConstantBitsFromNode(Y, Y.getScalarValueSizeInBits(), ConstUndef,
ConstEltBits, /*AllowWholeUndefs=*/false)) {
- bool splatIsConst =
+ bool SplatIsConst =
X86::isConstantSplat(SplatOp, SplatVal, /*AllowPartialUndefs=*/false);
// Can still shorten the chain when constant folded with the matrix
- if (!AndOp->hasOneUse() && !splatIsConst)
+ if (!AndOp->hasOneUse() && !SplatIsConst)
return SDValue();
- if (!(splatIsConst || DAG.isSplatValue(SplatOp, /*AllowUndefs=*/false)) ||
+ if (!(SplatIsConst || DAG.isSplatValue(SplatOp, /*AllowUndefs=*/false)) ||
SplatOp.getScalarValueSizeInBits() != 8)
return SDValue();
// ANDs with constants are not folded away this far into lowering
SDValue NewMatrix;
- if (!splatIsConst) {
+ if (!SplatIsConst) {
NewMatrix = DAG.getNode(ISD::AND, DL, VT, SplatOp, Y);
} else {
SmallVector<SDValue> FoldedAnd;
@@ -62184,8 +62184,7 @@ static SDValue combineAndOnGF2P8AFFINEQBOperand(SDNode *N, const SDLoc &DL,
return SDValue();
}
-static SDValue combineGF2P8AFFINEQB(SDNode *N, SelectionDAG &DAG,
- TargetLowering::DAGCombinerInfo &DCI) {
+static SDValue combineGF2P8AFFINEQB(SDNode *N, SelectionDAG &DAG) {
EVT VT = N->getValueType(0);
// const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDLoc dl(N);
@@ -62795,7 +62794,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::VPMADD52H: return combineVPMADD52LH(N, DAG, DCI);
case X86ISD::KSHIFTL:
case X86ISD::KSHIFTR: return combineKSHIFT(N, DAG, DCI);
- case X86ISD::GF2P8AFFINEQB: return combineGF2P8AFFINEQB(N, DAG, DCI);
+ case X86ISD::GF2P8AFFINEQB: return combineGF2P8AFFINEQB(N, DAG);
case ISD::FP16_TO_FP: return combineFP16_TO_FP(N, DAG, Subtarget);
case ISD::STRICT_FP_EXTEND:
case ISD::FP_EXTEND: return combineFP_EXTEND(N, DAG, DCI, Subtarget);
>From 8d015e3ccb920a18e60944e21c31b0cf5f1cc76d Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Tue, 28 Apr 2026 09:09:42 +1000
Subject: [PATCH 7/9] Remove dead `TargetLowering` comment
A target lowering information line was commented out due to it being unused. It has been removed instead for clarity.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9131cb97bdeb8..3def001b2f674 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -62186,7 +62186,6 @@ static SDValue combineAndOnGF2P8AFFINEQBOperand(SDNode *N, const SDLoc &DL,
static SDValue combineGF2P8AFFINEQB(SDNode *N, SelectionDAG &DAG) {
EVT VT = N->getValueType(0);
- // const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDLoc dl(N);
if (SDValue R = combineAndOnGF2P8AFFINEQBOperand(N, dl, DAG, VT))
>From 31c0641d65554a13736c1eea5df98e6810891774 Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Wed, 13 May 2026 21:54:24 +1000
Subject: [PATCH 8/9] Added constant pool comments
- Upstream added comments to gf2p8affine pool constant/memory operand. I updated the regression test to reflect that change.
---
llvm/test/CodeGen/X86/gfni-operand-and-fold.ll | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
index 011136eb4e51a..c3883213ac743 100644
--- a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
@@ -6,7 +6,7 @@ declare <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
define <16 x i8> @test_const_splat_on_const_matrix(<16 x i8> %src) nounwind {
; CHECK-LABEL: test_const_splat_on_const_matrix:
; CHECK: # %bb.0:
-; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0]
; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 15)
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 0)
@@ -16,7 +16,7 @@ define <16 x i8> @test_const_splat_on_const_matrix(<16 x i8> %src) nounwind {
define <32 x i8> @test_const_splat_on_const_matrix256(<32 x i8> %src) nounwind {
; CHECK-LABEL: test_const_splat_on_const_matrix256:
; CHECK: # %bb.0:
-; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0]
; CHECK-NEXT: retq
%and = and <32 x i8> %src, splat(i8 15)
%gfni = call <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8> %and, <32 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 0)
@@ -27,7 +27,7 @@ define <32 x i8> @test_const_splat_on_const_matrix256(<32 x i8> %src) nounwind {
define <16 x i8> @test_const_splat_alternative_const_matrix(<16 x i8> %src) nounwind {
; CHECK-LABEL: test_const_splat_alternative_const_matrix:
; CHECK: # %bb.0:
-; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [1,1,5,5,21,21,85,85,1,1,5,5,21,21,85,85]
; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 85)
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 1, i8 3, i8 7, i8 15, i8 31, i8 63, i8 127, i8 255, i8 1, i8 3, i8 7, i8 15, i8 31, i8 63, i8 127, i8 255>, i8 0)
@@ -37,7 +37,7 @@ define <16 x i8> @test_const_splat_alternative_const_matrix(<16 x i8> %src) noun
define <16 x i8> @test_const_splat_on_const_matrix_nonzero_imm(<16 x i8> %src) nounwind {
; CHECK-LABEL: test_const_splat_on_const_matrix_nonzero_imm:
; CHECK: # %bb.0:
-; CHECK-NEXT: vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0]
; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 15)
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 -128>, i8 127)
@@ -93,7 +93,7 @@ define <16 x i8> @test_const_splat_on_const_matrix_multi_use(<16 x i8> %src) nou
; CHECK-LABEL: test_const_splat_on_const_matrix_multi_use:
; CHECK: # %bb.0:
; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0]
; CHECK-NEXT: vpxor %xmm1, %xmm0, %xmm0
; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 15)
@@ -109,7 +109,7 @@ define <16 x i8> @test_var_splat_on_const_matrix_multi_use(<16 x i8> %src, i8 %s
; CHECK-NEXT: vmovd %edi, %xmm1
; CHECK-NEXT: vpbroadcastb %xmm1, %xmm1
; CHECK-NEXT: vpand %xmm1, %xmm0, %xmm0
-; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128]
; CHECK-NEXT: vpxor %xmm0, %xmm1, %xmm0
; CHECK-NEXT: retq
%inlo = insertelement <16 x i8> poison, i8 %scalar, i64 0
@@ -153,7 +153,7 @@ define <16 x i8> @test_const_splat16_on_const_matrix(<16 x i8> %src) nounwind {
; CHECK-LABEL: test_const_splat16_on_const_matrix:
; CHECK: # %bb.0:
; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128]
; CHECK-NEXT: retq
%splat = bitcast <8 x i16> splat(i16 1) to <16 x i8>
%and = and <16 x i8> %src, %splat
@@ -168,7 +168,7 @@ define <16 x i8> @test_var_splat16_on_const_matrix(<16 x i8> %src, i16 %scalar)
; CHECK-NEXT: vmovd %edi, %xmm1
; CHECK-NEXT: vpbroadcastw %xmm1, %xmm1
; CHECK-NEXT: vpand %xmm1, %xmm0, %xmm0
-; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128]
; CHECK-NEXT: retq
%inlo = insertelement <8 x i16> poison, i16 %scalar, i64 0
%varsplat16 = shufflevector <8 x i16> %inlo, <8 x i16> poison, <8 x i32> zeroinitializer
>From 40198bf78eee75006cde2b33e98e87ef252f3175 Mon Sep 17 00:00:00 2001
From: Walter <walter.kruger at hotmail.com>
Date: Wed, 13 May 2026 22:47:41 +1000
Subject: [PATCH 9/9] Safer multi-use test
Changed the multi-use regression test with a example where the AND is needed by something unrelated to the GF2P8AFFINEQB.
This prevents the possibility of it being completely folded away by further optimizations (`GFAFFINE(x, M) ^ x => GFAFFINE(x, M ^ IDENTITY)`). Its also clearer.
---
llvm/test/CodeGen/X86/gfni-operand-and-fold.ll | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
index c3883213ac743..a16771baaff7c 100644
--- a/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-operand-and-fold.ll
@@ -89,17 +89,17 @@ define <16 x i8> @test_var_splat_on_const_matrix_nonzero_imm(<16 x i8> %src, i8
}
;; Multi use still shortens dependency chain
-define <16 x i8> @test_const_splat_on_const_matrix_multi_use(<16 x i8> %src) nounwind {
+define <16 x i8> @test_const_splat_on_const_matrix_multi_use(<16 x i8> %src, ptr %sink) nounwind {
; CHECK-LABEL: test_const_splat_on_const_matrix_multi_use:
; CHECK: # %bb.0:
; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; CHECK-NEXT: vmovdqa %xmm1, (%rdi)
; CHECK-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0]
-; CHECK-NEXT: vpxor %xmm1, %xmm0, %xmm0
; CHECK-NEXT: retq
%and = and <16 x i8> %src, splat(i8 15)
+ store <16 x i8> %and, ptr %sink
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %and, <16 x i8> <i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128>, i8 0)
- %xor = xor <16 x i8> %gfni, %and
- ret <16 x i8> %xor
+ ret <16 x i8> %gfni
}
;; Negative test: multi use would generate an additional AND
More information about the llvm-commits
mailing list