[llvm] [AVX-512] Fix for disjoint-or-fold (VGF2P8AFFINEQB) (PR #190896)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 00:13:23 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/190896
>From eb4edef12b354336db8c85e1dd575cb1b2a08c32 Mon Sep 17 00:00:00 2001
From: kartikohlan <kartik7ohlan at gmail.com>
Date: Tue, 7 Apr 2026 22:48:39 -0400
Subject: [PATCH 1/5] Fixed the formatting
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 28 +++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index ebc8766969251..eb65710820fd9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -53171,6 +53171,31 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, const SDLoc &DL,
return SDValue();
}
+static SDValue combineOrWithGF2P8AFFINEQB(SDNode *N, const SDLoc &DL,
+ SelectionDAG &DAG, EVT VT) {
+ using namespace SDPatternMatch;
+ assert(N->getOpcode() == ISD::OR);
+
+ if (!N->getFlags().hasDisjoint() &&
+ !DAG.haveNoCommonBitsSet(N->getOperand(0), N->getOperand(1)))
+ return SDValue();
+
+ SDValue X, Y, SplatOp;
+ APInt Imm, SplatVal;
+ if (sd_match(N,
+ m_c_BinOp(ISD::OR,
+ m_OneUse(m_TernaryOp(X86ISD::GF2P8AFFINEQB, m_Value(X),
+ m_Value(Y), m_ConstInt(Imm))),
+ m_Value(SplatOp))) &&
+ X86::isConstantSplat(SplatOp, SplatVal, /*AllowPartialUndefs=*/false)) {
+ uint64_t NewImm = (Imm.getZExtValue() ^ SplatVal.getZExtValue()) & 0xFF;
+ return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, X, Y,
+ DAG.getTargetConstant(NewImm, DL, MVT::i8));
+ }
+
+ return SDValue();
+}
+
static SDValue combineOrXorWithSETCC(unsigned Opc, const SDLoc &DL, EVT VT,
SDValue N0, SDValue N1,
SelectionDAG &DAG) {
@@ -53263,6 +53288,9 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
DAG, DCI, Subtarget))
return FPLogic;
+ if (SDValue R = combineOrWithGF2P8AFFINEQB(N, dl, DAG, VT))
+ return R;
+
if (DCI.isBeforeLegalizeOps())
return SDValue();
>From e3370b27bb87624a47b871ee9e27712d66312ad9 Mon Sep 17 00:00:00 2001
From: kartikohlan <kartik7ohlan at gmail.com>
Date: Wed, 8 Apr 2026 01:21:24 -0400
Subject: [PATCH 2/5] tests
---
llvm/test/CodeGen/X86/gfni-or-fold.ll | 38 +++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/gfni-or-fold.ll
diff --git a/llvm/test/CodeGen/X86/gfni-or-fold.ll b/llvm/test/CodeGen/X86/gfni-or-fold.ll
new file mode 100644
index 0000000000000..19029e69e54e3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/gfni-or-fold.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512bw,+avx512f,+gfni | FileCheck %s --check-prefixes=AVX512
+
+declare <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8>, <64 x i8>, i8)
+declare <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
+
+define <64 x i8> @test_or_disjoint_fold_512(<64 x i8> %src, <64 x i8> %matrix) nounwind {
+; AVX512-LABEL: test_or_disjoint_fold_512:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vgf2p8affineqb $0, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src, <64 x i8> %matrix, i8 0)
+ %or = or disjoint <64 x i8> %gfni, splat(i8 8)
+ ret <64 x i8> %or
+}
+
+define <64 x i8> @test_or_disjoint_fold_nonzero_imm(<64 x i8> %src, <64 x i8> %matrix) nounwind {
+; AVX512-LABEL: test_or_disjoint_fold_nonzero_imm:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vgf2p8affineqb $4, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src, <64 x i8> %matrix, i8 4)
+ %or = or disjoint <64 x i8> %gfni, splat(i8 8)
+ ret <64 x i8> %or
+}
+
+define <16 x i8> @test_or_disjoint_fold_128(<16 x i8> %src, <16 x i8> %matrix) nounwind {
+; AVX512-LABEL: test_or_disjoint_fold_128:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %src, <16 x i8> %matrix, i8 0)
+ %or = or disjoint <16 x i8> %gfni, splat(i8 8)
+ ret <16 x i8> %or
+}
>From 4ece88d07f2779891069bc1597eb4beb60ca754e Mon Sep 17 00:00:00 2001
From: kartikohlan <kartik7ohlan at gmail.com>
Date: Wed, 8 Apr 2026 18:29:38 -0400
Subject: [PATCH 3/5] Fixed the x86 linking issue
---
llvm/test/CodeGen/X86/gfni-or-fold.ll | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/X86/gfni-or-fold.ll b/llvm/test/CodeGen/X86/gfni-or-fold.ll
index 19029e69e54e3..346090f6e96c2 100644
--- a/llvm/test/CodeGen/X86/gfni-or-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-or-fold.ll
@@ -7,8 +7,7 @@ declare <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
define <64 x i8> @test_or_disjoint_fold_512(<64 x i8> %src, <64 x i8> %matrix) nounwind {
; AVX512-LABEL: test_or_disjoint_fold_512:
; AVX512: # %bb.0:
-; AVX512-NEXT: vgf2p8affineqb $0, %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512-NEXT: vgf2p8affineqb $8, %zmm1, %zmm0, %zmm0
; AVX512-NEXT: retq
%gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src, <64 x i8> %matrix, i8 0)
%or = or disjoint <64 x i8> %gfni, splat(i8 8)
@@ -18,8 +17,7 @@ define <64 x i8> @test_or_disjoint_fold_512(<64 x i8> %src, <64 x i8> %matrix) n
define <64 x i8> @test_or_disjoint_fold_nonzero_imm(<64 x i8> %src, <64 x i8> %matrix) nounwind {
; AVX512-LABEL: test_or_disjoint_fold_nonzero_imm:
; AVX512: # %bb.0:
-; AVX512-NEXT: vgf2p8affineqb $4, %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512-NEXT: vgf2p8affineqb $12, %zmm1, %zmm0, %zmm0
; AVX512-NEXT: retq
%gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src, <64 x i8> %matrix, i8 4)
%or = or disjoint <64 x i8> %gfni, splat(i8 8)
@@ -29,8 +27,7 @@ define <64 x i8> @test_or_disjoint_fold_nonzero_imm(<64 x i8> %src, <64 x i8> %m
define <16 x i8> @test_or_disjoint_fold_128(<16 x i8> %src, <16 x i8> %matrix) nounwind {
; AVX512-LABEL: test_or_disjoint_fold_128:
; AVX512: # %bb.0:
-; AVX512-NEXT: vgf2p8affineqb $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: vgf2p8affineqb $8, %xmm1, %xmm0, %xmm0
; AVX512-NEXT: retq
%gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %src, <16 x i8> %matrix, i8 0)
%or = or disjoint <16 x i8> %gfni, splat(i8 8)
>From c0ba9a578234887d7c1f6bad40edd7de0192c794 Mon Sep 17 00:00:00 2001
From: kartikohlan <kartik7ohlan at gmail.com>
Date: Thu, 9 Apr 2026 09:31:22 -0400
Subject: [PATCH 4/5] Added negative test cases
---
llvm/test/CodeGen/X86/gfni-or-fold.ll | 36 +++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/llvm/test/CodeGen/X86/gfni-or-fold.ll b/llvm/test/CodeGen/X86/gfni-or-fold.ll
index 346090f6e96c2..1a9e4c11518b2 100644
--- a/llvm/test/CodeGen/X86/gfni-or-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-or-fold.ll
@@ -33,3 +33,39 @@ define <16 x i8> @test_or_disjoint_fold_128(<16 x i8> %src, <16 x i8> %matrix) n
%or = or disjoint <16 x i8> %gfni, splat(i8 8)
ret <16 x i8> %or
}
+
+define <64 x i8> @test_or_no_fold_not_disjoint(<64 x i8> %src, <64 x i8> %matrix) nounwind {
+; AVX512-LABEL: test_or_no_fold_not_disjoint:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vgf2p8affineqb $8, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src, <64 x i8> %matrix, i8 8)
+ %or = or <64 x i8> %gfni, splat(i8 8)
+ ret <64 x i8> %or
+}
+
+define <64 x i8> @test_or_non_disjoint(<64 x i8> %src, <64 x i8> %matrix) {
+; AVX512-LABEL: test_or_non_disjoint:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vgf2p8affineqb $4, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src, <64 x i8> %matrix, i8 4)
+ ; Without the 'disjoint' keyword, the compiler cannot safely fold this into the immediate
+ %or = or <64 x i8> %gfni, splat(i8 4)
+ ret <64 x i8> %or
+}
+
+define <64 x i8> @test_or_no_fold_multi_use(<64 x i8> %src, <64 x i8> %matrix, ptr %out) nounwind {
+; AVX512-LABEL: test_or_no_fold_multi_use:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vgf2p8affineqb $0, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vmovdqa64 %zmm0, (%rdi)
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src, <64 x i8> %matrix, i8 0)
+ store <64 x i8> %gfni, ptr %out ; second use
+ %or = or disjoint <64 x i8> %gfni, splat(i8 8)
+ ret <64 x i8> %or
+}
>From 438199668623f7ee2c76bf2edf19750ba4323e83 Mon Sep 17 00:00:00 2001
From: Kartik Ohlan <kartik7ohlan at gmail.com>
Date: Sun, 12 Apr 2026 15:25:51 -0400
Subject: [PATCH 5/5] Cleared final nits and added test for immediate overalap
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 16 ++++++++++------
llvm/test/CodeGen/X86/gfni-or-fold.ll | 10 ++++++++++
2 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index eb65710820fd9..a2e2d90043625 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -53174,7 +53174,7 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, const SDLoc &DL,
static SDValue combineOrWithGF2P8AFFINEQB(SDNode *N, const SDLoc &DL,
SelectionDAG &DAG, EVT VT) {
using namespace SDPatternMatch;
- assert(N->getOpcode() == ISD::OR);
+ assert(N->getOpcode() == ISD::OR && "Expected OR node");
if (!N->getFlags().hasDisjoint() &&
!DAG.haveNoCommonBitsSet(N->getOperand(0), N->getOperand(1)))
@@ -53182,11 +53182,15 @@ static SDValue combineOrWithGF2P8AFFINEQB(SDNode *N, const SDLoc &DL,
SDValue X, Y, SplatOp;
APInt Imm, SplatVal;
- if (sd_match(N,
- m_c_BinOp(ISD::OR,
- m_OneUse(m_TernaryOp(X86ISD::GF2P8AFFINEQB, m_Value(X),
- m_Value(Y), m_ConstInt(Imm))),
- m_Value(SplatOp))) &&
+
+ // Fold: (GF2P8AFFINEQB(X, Y, Imm) or_disjoint SplatVal)
+ // -> GF2P8AFFINEQB(X, Y, Imm ^ SplatVal)
+ // When OR is disjoint (no common bits), the splat constant can be folded
+ // directly into the GF2P8AFFINEQB immediate via XOR.
+
+ if (sd_match(N, m_Or(m_OneUse(m_TernaryOp(X86ISD::GF2P8AFFINEQB, m_Value(X),
+ m_Value(Y), m_ConstInt(Imm))),
+ m_Value(SplatOp))) &&
X86::isConstantSplat(SplatOp, SplatVal, /*AllowPartialUndefs=*/false)) {
uint64_t NewImm = (Imm.getZExtValue() ^ SplatVal.getZExtValue()) & 0xFF;
return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, X, Y,
diff --git a/llvm/test/CodeGen/X86/gfni-or-fold.ll b/llvm/test/CodeGen/X86/gfni-or-fold.ll
index 1a9e4c11518b2..62cd56e223b9f 100644
--- a/llvm/test/CodeGen/X86/gfni-or-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-or-fold.ll
@@ -69,3 +69,13 @@ define <64 x i8> @test_or_no_fold_multi_use(<64 x i8> %src, <64 x i8> %matrix, p
%or = or disjoint <64 x i8> %gfni, splat(i8 8)
ret <64 x i8> %or
}
+
+define <64 x i8> @test_or_disjoint_overlap_with_imm(<64 x i8> %src, <64 x i8> %matrix) nounwind {
+; AVX512-LABEL: test_or_disjoint_overlap_with_imm:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vgf2p8affineqb $0, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src, <64 x i8> %matrix, i8 8)
+ %or = or disjoint <64 x i8> %gfni, splat(i8 8)
+ ret <64 x i8> %or
+}
More information about the llvm-commits
mailing list