[llvm] [AMDGPU] Fix VOP3P NEG fold to check the use opcode, not the def (PR #219728)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 01:55:26 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/219728
>From b379af03c8ae8bfb75e199a5fda0855e20f92604 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Sat, 29 Aug 2026 23:34:06 +0200
Subject: [PATCH 1/2] [AMDGPU] Fix VOP3P NEG fold to check the use opcode, not
the def
The GISel def type loses FP/int distinction across a same-size G_BITCAST, so an FP fneg could fold into an integer packed op that ignores NEG bits
Base the check on the consuming instruction instead
---
.../AMDGPU/AMDGPUInstructionSelector.cpp | 64 +++++++++++-------
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 4 +-
.../CodeGen/AMDGPU/GlobalISel/add.v2i16.ll | 20 ++++--
.../AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll | 12 ++--
.../AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll | 18 +++--
.../CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll | 20 ++++--
.../CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll | 30 ++++++---
.../AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll | 65 +++++++++++++++++++
8 files changed, 178 insertions(+), 55 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 353506378b229..b64584ecdfd4a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -24,6 +24,7 @@
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -5163,9 +5164,37 @@ calcNextStatus(std::pair<Register, SrcStatus> Curr,
return std::nullopt;
}
-/// This is used to control valid status that current MI supports. For example,
-/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
-/// bit on VOP3P.
+/// Packed integer VOP3P opcodes ignore NEG/NEG_HI, so folding a G_FNEG into
+/// them would silently drop the negation.
+static bool usesFPSrcMods(const MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ if (isPreISelGenericFloatingPointOpcode(Opc))
+ return true;
+
+ // To re-audit, collect the direct parent of each "(VOP3PMods" in
+ // AMDGPUGenGlobalISel.inc.
+ switch (Opc) {
+ case TargetOpcode::G_STRICT_FADD:
+ case TargetOpcode::G_STRICT_FMUL:
+ case TargetOpcode::G_STRICT_FMA:
+ case AMDGPU::G_AMDGPU_CLAMP:
+ case AMDGPU::G_AMDGPU_FMIN3:
+ case AMDGPU::G_AMDGPU_FMAX3:
+ case AMDGPU::G_AMDGPU_FMINIMUM3:
+ case AMDGPU::G_AMDGPU_FMAXIMUM3:
+ return true;
+ case TargetOpcode::G_INTRINSIC: {
+ // The only dot products selecting to a VOP3P with NEG bits.
+ Intrinsic::ID ID = cast<GIntrinsic>(MI).getIntrinsicID();
+ return ID == Intrinsic::amdgcn_fdot2 ||
+ ID == Intrinsic::amdgcn_fdot2_f32_bf16;
+ }
+ default:
+ return false;
+ }
+}
+
+/// Controls which SrcStatus values the user of the folded value supports.
/// The class can be further extended to recognize support on SEL, NEG, ABS bit
/// for different MI on different arch
class SearchOptions {
@@ -5175,20 +5204,8 @@ class SearchOptions {
bool HasOpsel = true;
public:
- SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
- const MachineInstr *MI = MRI.getVRegDef(Reg);
- unsigned Opc = MI->getOpcode();
-
- if (Opc == TargetOpcode::G_INTRINSIC) {
- Intrinsic::ID IntrinsicID = cast<GIntrinsic>(*MI).getIntrinsicID();
- // Only float point intrinsic has neg & neg_hi bits.
- if (IntrinsicID == Intrinsic::amdgcn_fdot2)
- HasNeg = true;
- } else if (TargetInstrInfo::isGenericOpcode(Opc)) {
- // Keep same for generic op.
- HasNeg = true;
- }
- }
+ explicit SearchOptions(const MachineInstr &UseMI)
+ : HasNeg(usesFPSrcMods(UseMI)) {}
bool checkOptions(SrcStatus Stat) const {
if (!HasNeg &&
(Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
@@ -5283,8 +5300,11 @@ static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
IsHalfState(HiStat);
}
-std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
- Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const {
+std::pair<Register, unsigned>
+AMDGPUInstructionSelector::selectVOP3PModsImpl(const MachineOperand &Root,
+ const MachineRegisterInfo &MRI,
+ bool IsDOT) const {
+ Register RootReg = Root.getReg();
unsigned Mods = 0;
// No modification if Root type is not form of <2 x Type>.
if (isVectorOfTwoOrScalar(RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
@@ -5292,7 +5312,7 @@ std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
return {RootReg, Mods};
}
- SearchOptions SO(RootReg, MRI);
+ SearchOptions SO(*Root.getParent());
std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(RootReg, MRI, SO);
@@ -5395,7 +5415,7 @@ AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
Register Reg;
unsigned Mods;
- std::tie(Reg, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, IsDOT);
+ std::tie(Reg, Mods) = selectVOP3PModsImpl(Root, MRI, IsDOT);
Reg = getLegalRegBank(Reg, Root.getReg(), *Root.getParent(), RBI, MRI, TRI,
TII);
@@ -5422,7 +5442,7 @@ AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
Register Src;
unsigned Mods;
- std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true /*IsDOT*/);
+ std::tie(Src, Mods) = selectVOP3PModsImpl(Root, MRI, true /*IsDOT*/);
if (Mods != SISrcMods::OP_SEL_1)
return {};
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index ce3a090651d0e..1364954005023 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -190,8 +190,8 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
std::pair<Register, unsigned>
- selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,
- bool IsDOT = false) const;
+ selectVOP3PModsImpl(const MachineOperand &Root,
+ const MachineRegisterInfo &MRI, bool IsDOT = false) const;
InstructionSelector::ComplexRendererFns
selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
index a050fc43b453c..9f9f58463efe0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
@@ -60,7 +60,8 @@ define <2 x i16> @v_add_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX9-LABEL: v_add_v2i16_fneg_lhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[1,0] neg_hi:[1,0]
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_pk_add_u16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_add_v2i16_fneg_lhs:
@@ -75,7 +76,8 @@ define <2 x i16> @v_add_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX10-LABEL: v_add_v2i16_fneg_lhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[1,0] neg_hi:[1,0]
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_pk_add_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
%cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -101,7 +103,8 @@ define <2 x i16> @v_add_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX9-LABEL: v_add_v2i16_fneg_rhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: v_pk_add_u16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_add_v2i16_fneg_rhs:
@@ -116,7 +119,8 @@ define <2 x i16> @v_add_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX10-LABEL: v_add_v2i16_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_pk_add_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.b = fneg <2 x half> %b
%cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
@@ -143,7 +147,9 @@ define <2 x i16> @v_add_v2i16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX9-LABEL: v_add_v2i16_fneg_lhs_fneg_rhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[1,1] neg_hi:[1,1]
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: v_pk_add_u16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_add_v2i16_fneg_lhs_fneg_rhs:
@@ -159,7 +165,9 @@ define <2 x i16> @v_add_v2i16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX10-LABEL: v_add_v2i16_fneg_lhs_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[1,1] neg_hi:[1,1]
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_pk_add_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
%neg.b = fneg <2 x half> %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
index bc0de08726035..62ef9ffa32a92 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
@@ -196,7 +196,8 @@ define i32 @v_sdot2_fneg_a(<2 x half> %a, <2 x i16> %b, i32 %c) {
; GFX906-LABEL: v_sdot2_fneg_a:
; GFX906: ; %bb.0:
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_dot2_i32_i16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0]
+; GFX906-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX906-NEXT: v_dot2_i32_i16 v0, v0, v1, v2
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: v_sdot2_fneg_a:
@@ -210,7 +211,8 @@ define i32 @v_sdot2_fneg_a(<2 x half> %a, <2 x i16> %b, i32 %c) {
; GFX10-LABEL: v_sdot2_fneg_a:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_dot2_i32_i16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0]
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_dot2_i32_i16 v0, v0, v1, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
%cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -222,7 +224,8 @@ define i32 @v_sdot2_fneg_b(<2 x i16> %a, <2 x half> %b, i32 %c) {
; GFX906-LABEL: v_sdot2_fneg_b:
; GFX906: ; %bb.0:
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_dot2_i32_i16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0]
+; GFX906-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX906-NEXT: v_dot2_i32_i16 v0, v0, v1, v2
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: v_sdot2_fneg_b:
@@ -236,7 +239,8 @@ define i32 @v_sdot2_fneg_b(<2 x i16> %a, <2 x half> %b, i32 %c) {
; GFX10-LABEL: v_sdot2_fneg_b:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_dot2_i32_i16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0]
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_dot2_i32_i16 v0, v0, v1, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.b = fneg <2 x half> %b
%cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
index e287f78b9e352..00244c9374423 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
@@ -187,19 +187,22 @@ define i32 @v_udot2_fneg_a(<2 x half> %a, <2 x i16> %b, i32 %c) {
; GFX906-LABEL: v_udot2_fneg_a:
; GFX906: ; %bb.0:
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_dot2_u32_u16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0]
+; GFX906-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX906-NEXT: v_dot2_u32_u16 v0, v0, v1, v2
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: v_udot2_fneg_a:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_dot2_u32_u16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0]
+; GFX908-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX908-NEXT: v_dot2_u32_u16 v0, v0, v1, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_udot2_fneg_a:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_dot2_u32_u16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0]
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_dot2_u32_u16 v0, v0, v1, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
%cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -211,19 +214,22 @@ define i32 @v_udot2_fneg_b(<2 x i16> %a, <2 x half> %b, i32 %c) {
; GFX906-LABEL: v_udot2_fneg_b:
; GFX906: ; %bb.0:
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_dot2_u32_u16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0]
+; GFX906-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX906-NEXT: v_dot2_u32_u16 v0, v0, v1, v2
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: v_udot2_fneg_b:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_dot2_u32_u16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0]
+; GFX908-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX908-NEXT: v_dot2_u32_u16 v0, v0, v1, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_udot2_fneg_b:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_dot2_u32_u16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0]
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_dot2_u32_u16 v0, v0, v1, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.b = fneg <2 x half> %b
%cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll
index 6755a2faf2950..f06f30978f990 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll
@@ -32,7 +32,8 @@ define <2 x i16> @v_mul_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX9-LABEL: v_mul_v2i16_fneg_lhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1 neg_lo:[1,0] neg_hi:[1,0]
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_v2i16_fneg_lhs:
@@ -47,7 +48,8 @@ define <2 x i16> @v_mul_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX10-LABEL: v_mul_v2i16_fneg_lhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1 neg_lo:[1,0] neg_hi:[1,0]
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
%cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -59,7 +61,8 @@ define <2 x i16> @v_mul_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX9-LABEL: v_mul_v2i16_fneg_rhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_v2i16_fneg_rhs:
@@ -74,7 +77,8 @@ define <2 x i16> @v_mul_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX10-LABEL: v_mul_v2i16_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.b = fneg <2 x half> %b
%cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
@@ -86,7 +90,9 @@ define <2 x i16> @v_mul_v2i16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX9-LABEL: v_mul_v2i16_fneg_lhs_fneg_rhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1 neg_lo:[1,1] neg_hi:[1,1]
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_v2i16_fneg_lhs_fneg_rhs:
@@ -102,7 +108,9 @@ define <2 x i16> @v_mul_v2i16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX10-LABEL: v_mul_v2i16_fneg_lhs_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1 neg_lo:[1,1] neg_hi:[1,1]
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
%neg.b = fneg <2 x half> %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
index 195a3a623f0d6..5bb20b8a30d72 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
@@ -38,7 +38,8 @@ define <2 x i16> @v_sub_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX9-LABEL: v_sub_v2i16_fneg_lhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[1,0] neg_hi:[1,0]
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sub_v2i16_fneg_lhs:
@@ -53,13 +54,15 @@ define <2 x i16> @v_sub_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX10-LABEL: v_sub_v2i16_fneg_lhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[1,0] neg_hi:[1,0]
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_sub_v2i16_fneg_lhs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[1,0] neg_hi:[1,0]
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX11-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
%cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -71,7 +74,8 @@ define <2 x i16> @v_sub_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX9-LABEL: v_sub_v2i16_fneg_rhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sub_v2i16_fneg_rhs:
@@ -86,13 +90,15 @@ define <2 x i16> @v_sub_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX10-LABEL: v_sub_v2i16_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_sub_v2i16_fneg_rhs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX11-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX11-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%neg.b = fneg <2 x half> %b
%cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
@@ -104,7 +110,9 @@ define <2 x i16> @v_sub_v2i16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX9-LABEL: v_sub_v2i16_fneg_lhs_fneg_rhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[1,1] neg_hi:[1,1]
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sub_v2i16_fneg_lhs_fneg_rhs:
@@ -120,13 +128,17 @@ define <2 x i16> @v_sub_v2i16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX10-LABEL: v_sub_v2i16_fneg_lhs_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[1,1] neg_hi:[1,1]
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_sub_v2i16_fneg_lhs_fneg_rhs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_sub_i16 v0, v0, v1 neg_lo:[1,1] neg_hi:[1,1]
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX11-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX11-NEXT: v_pk_sub_i16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
%neg.b = fneg <2 x half> %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll
new file mode 100644
index 0000000000000..618b0eb8ac5c9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
+
+; An fneg bitcast to <2 x i16> and fed to a packed integer op must select as a
+; real xor: neg_lo/neg_hi only apply to FP. add/sub/mul and the integer dot
+; products are covered in {add,sub,mul}.v2i16.ll and llvm.amdgcn.{s,u}dot2.ll.
+
+define <2 x i16> @pk_smax_v2i16_fneg_lhs(<2 x half> %x, <2 x i16> %y) {
+; GFX9-LABEL: pk_smax_v2i16_fneg_lhs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_pk_max_i16 v0, v0, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: pk_smax_v2i16_fneg_lhs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX11-NEXT: v_pk_max_i16 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %n = fneg <2 x half> %x
+ %i = bitcast <2 x half> %n to <2 x i16>
+ %r = call <2 x i16> @llvm.smax.v2i16(<2 x i16> %i, <2 x i16> %y)
+ ret <2 x i16> %r
+}
+
+define <2 x i16> @pk_umin_v2i16_fneg_rhs(<2 x half> %x, <2 x i16> %y) {
+; GFX9-LABEL: pk_umin_v2i16_fneg_rhs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_pk_min_u16 v0, v1, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: pk_umin_v2i16_fneg_rhs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX11-NEXT: v_pk_min_u16 v0, v1, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %n = fneg <2 x half> %x
+ %i = bitcast <2 x half> %n to <2 x i16>
+ %r = call <2 x i16> @llvm.umin.v2i16(<2 x i16> %y, <2 x i16> %i)
+ ret <2 x i16> %r
+}
+
+; The fold must be preserved for packed FP users.
+define <2 x half> @pk_add_v2f16_fneg_lhs(<2 x half> %x, <2 x half> %y) {
+; GFX9-LABEL: pk_add_v2f16_fneg_lhs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_pk_add_f16 v0, v1, v0 neg_lo:[0,1] neg_hi:[0,1]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: pk_add_v2f16_fneg_lhs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_pk_add_f16 v0, v1, v0 neg_lo:[0,1] neg_hi:[0,1]
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %n = fneg <2 x half> %x
+ %r = fadd <2 x half> %n, %y
+ ret <2 x half> %r
+}
>From 866dfe8272395ea0653d63dc1efea1dc04bbfd9f Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 8 Sep 2026 10:55:14 +0200
Subject: [PATCH 2/2] Address review comments
---
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 3 +++
.../AMDGPU/AMDGPUInstructionSelector.cpp | 19 ++++++++-----------
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 4 ++--
.../AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll | 4 ++--
4 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index c9f5c6248d1f1..18a36279b2d11 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1709,6 +1709,9 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
case TargetOpcode::G_FLOG:
case TargetOpcode::G_FMA:
case TargetOpcode::G_FMAD:
+ case TargetOpcode::G_STRICT_FADD:
+ case TargetOpcode::G_STRICT_FMUL:
+ case TargetOpcode::G_STRICT_FMA:
case TargetOpcode::G_FMAXIMUM:
case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_FMAXNUM:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index b64584ecdfd4a..5b7267fca7ac4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -5174,9 +5174,6 @@ static bool usesFPSrcMods(const MachineInstr &MI) {
// To re-audit, collect the direct parent of each "(VOP3PMods" in
// AMDGPUGenGlobalISel.inc.
switch (Opc) {
- case TargetOpcode::G_STRICT_FADD:
- case TargetOpcode::G_STRICT_FMUL:
- case TargetOpcode::G_STRICT_FMA:
case AMDGPU::G_AMDGPU_CLAMP:
case AMDGPU::G_AMDGPU_FMIN3:
case AMDGPU::G_AMDGPU_FMAX3:
@@ -5300,11 +5297,9 @@ static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
IsHalfState(HiStat);
}
-std::pair<Register, unsigned>
-AMDGPUInstructionSelector::selectVOP3PModsImpl(const MachineOperand &Root,
- const MachineRegisterInfo &MRI,
- bool IsDOT) const {
- Register RootReg = Root.getReg();
+std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
+ Register RootReg, const MachineRegisterInfo &MRI, const MachineInstr &UseMI,
+ bool IsDOT) const {
unsigned Mods = 0;
// No modification if Root type is not form of <2 x Type>.
if (isVectorOfTwoOrScalar(RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
@@ -5312,7 +5307,7 @@ AMDGPUInstructionSelector::selectVOP3PModsImpl(const MachineOperand &Root,
return {RootReg, Mods};
}
- SearchOptions SO(*Root.getParent());
+ SearchOptions SO(UseMI);
std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(RootReg, MRI, SO);
@@ -5415,7 +5410,8 @@ AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
Register Reg;
unsigned Mods;
- std::tie(Reg, Mods) = selectVOP3PModsImpl(Root, MRI, IsDOT);
+ std::tie(Reg, Mods) =
+ selectVOP3PModsImpl(Root.getReg(), MRI, *Root.getParent(), IsDOT);
Reg = getLegalRegBank(Reg, Root.getReg(), *Root.getParent(), RBI, MRI, TRI,
TII);
@@ -5442,7 +5438,8 @@ AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
Register Src;
unsigned Mods;
- std::tie(Src, Mods) = selectVOP3PModsImpl(Root, MRI, true /*IsDOT*/);
+ std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI,
+ *Root.getParent(), true /*IsDOT*/);
if (Mods != SISrcMods::OP_SEL_1)
return {};
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 1364954005023..14ea80332021d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -190,8 +190,8 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
std::pair<Register, unsigned>
- selectVOP3PModsImpl(const MachineOperand &Root,
- const MachineRegisterInfo &MRI, bool IsDOT = false) const;
+ selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,
+ const MachineInstr &UseMI, bool IsDOT = false) const;
InstructionSelector::ComplexRendererFns
selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll
index 618b0eb8ac5c9..905226d36d7d2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/v2i16-fneg-src-mods.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
; An fneg bitcast to <2 x i16> and fed to a packed integer op must select as a
; real xor: neg_lo/neg_hi only apply to FP. add/sub/mul and the integer dot
More information about the llvm-commits
mailing list