[llvm] 286d3fc - [AMDGPU] Split R600 and GCN bfi patterns
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 02:17:12 PDT 2020
Author: Jay Foad
Date: 2020-09-28T10:16:51+01:00
New Revision: 286d3fc7500dfa6a947b217d0e87d1764d2242b8
URL: https://github.com/llvm/llvm-project/commit/286d3fc7500dfa6a947b217d0e87d1764d2242b8
DIFF: https://github.com/llvm/llvm-project/commit/286d3fc7500dfa6a947b217d0e87d1764d2242b8.diff
LOG: [AMDGPU] Split R600 and GCN bfi patterns
This is in preparation for making the GCN patterns divergence-aware.
NFC.
Differential Revision: https://reviews.llvm.org/D88244
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
llvm/lib/Target/AMDGPU/EvergreenInstructions.td
llvm/lib/Target/AMDGPU/SIInstructions.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 01c7934e9eb0..23e47c6cc14d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -597,102 +597,6 @@ class DwordAddrPat<ValueType vt, RegisterClass rc> : AMDGPUPat <
(vt rc:$addr)
>;
-// BFI_INT patterns
-
-multiclass BFIPatterns <Instruction BFI_INT,
- Instruction LoadImm32,
- RegisterClass RC64> {
- // Definition from ISA doc:
- // (y & x) | (z & ~x)
- def : AMDGPUPat <
- (or (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))),
- (BFI_INT $x, $y, $z)
- >;
-
- // 64-bit version
- def : AMDGPUPat <
- (or (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))),
- (REG_SEQUENCE RC64,
- (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
- (i32 (EXTRACT_SUBREG RC64:$y, sub0)),
- (i32 (EXTRACT_SUBREG RC64:$z, sub0))), sub0,
- (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
- (i32 (EXTRACT_SUBREG RC64:$y, sub1)),
- (i32 (EXTRACT_SUBREG RC64:$z, sub1))), sub1)
- >;
-
- // SHA-256 Ch function
- // z ^ (x & (y ^ z))
- def : AMDGPUPat <
- (xor i32:$z, (and i32:$x, (xor i32:$y, i32:$z))),
- (BFI_INT $x, $y, $z)
- >;
-
- // 64-bit version
- def : AMDGPUPat <
- (xor i64:$z, (and i64:$x, (xor i64:$y, i64:$z))),
- (REG_SEQUENCE RC64,
- (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
- (i32 (EXTRACT_SUBREG RC64:$y, sub0)),
- (i32 (EXTRACT_SUBREG RC64:$z, sub0))), sub0,
- (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
- (i32 (EXTRACT_SUBREG RC64:$y, sub1)),
- (i32 (EXTRACT_SUBREG RC64:$z, sub1))), sub1)
- >;
-
- def : AMDGPUPat <
- (fcopysign f32:$src0, f32:$src1),
- (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0, $src1)
- >;
-
- def : AMDGPUPat <
- (f32 (fcopysign f32:$src0, f64:$src1)),
- (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0,
- (i32 (EXTRACT_SUBREG RC64:$src1, sub1)))
- >;
-
- def : AMDGPUPat <
- (f64 (fcopysign f64:$src0, f64:$src1)),
- (REG_SEQUENCE RC64,
- (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
- (BFI_INT (LoadImm32 (i32 0x7fffffff)),
- (i32 (EXTRACT_SUBREG RC64:$src0, sub1)),
- (i32 (EXTRACT_SUBREG RC64:$src1, sub1))), sub1)
- >;
-
- def : AMDGPUPat <
- (f64 (fcopysign f64:$src0, f32:$src1)),
- (REG_SEQUENCE RC64,
- (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
- (BFI_INT (LoadImm32 (i32 0x7fffffff)),
- (i32 (EXTRACT_SUBREG RC64:$src0, sub1)),
- $src1), sub1)
- >;
-}
-
-// SHA-256 Ma patterns
-
-// ((x & z) | (y & (x | z))) -> BFI_INT (XOR x, y), z, y
-multiclass SHA256MaPattern <Instruction BFI_INT, Instruction XOR, RegisterClass RC64> {
- def : AMDGPUPat <
- (or (and i32:$x, i32:$z), (and i32:$y, (or i32:$x, i32:$z))),
- (BFI_INT (XOR i32:$x, i32:$y), i32:$z, i32:$y)
- >;
-
- def : AMDGPUPat <
- (or (and i64:$x, i64:$z), (and i64:$y, (or i64:$x, i64:$z))),
- (REG_SEQUENCE RC64,
- (BFI_INT (XOR (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
- (i32 (EXTRACT_SUBREG RC64:$y, sub0))),
- (i32 (EXTRACT_SUBREG RC64:$z, sub0)),
- (i32 (EXTRACT_SUBREG RC64:$y, sub0))), sub0,
- (BFI_INT (XOR (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
- (i32 (EXTRACT_SUBREG RC64:$y, sub1))),
- (i32 (EXTRACT_SUBREG RC64:$z, sub1)),
- (i32 (EXTRACT_SUBREG RC64:$y, sub1))), sub1)
- >;
-}
-
// Bitfield extract patterns
def IMMZeroBasedBitfieldMask : ImmLeaf <i32, [{
diff --git a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
index 97104a242d8c..a2782bf8b67d 100644
--- a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
+++ b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
@@ -408,7 +408,74 @@ def : EGOrCaymanPat<(i32 (sext_inreg i32:$src, i8)),
def : EGOrCaymanPat<(i32 (sext_inreg i32:$src, i16)),
(BFE_INT_eg i32:$src, (i32 ZERO), (MOV_IMM_I32 16))>;
-defm : BFIPatterns <BFI_INT_eg, MOV_IMM_I32, R600_Reg64>;
+// BFI patterns
+
+// Definition from ISA doc:
+// (y & x) | (z & ~x)
+def : AMDGPUPat <
+ (or (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))),
+ (BFI_INT_eg $x, $y, $z)
+>;
+
+// 64-bit version
+def : AMDGPUPat <
+ (or (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))),
+ (REG_SEQUENCE R600_Reg64,
+ (BFI_INT_eg (i32 (EXTRACT_SUBREG R600_Reg64:$x, sub0)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$y, sub0)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$z, sub0))), sub0,
+ (BFI_INT_eg (i32 (EXTRACT_SUBREG R600_Reg64:$x, sub1)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$y, sub1)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$z, sub1))), sub1)
+>;
+
+// SHA-256 Ch function
+// z ^ (x & (y ^ z))
+def : AMDGPUPat <
+ (xor i32:$z, (and i32:$x, (xor i32:$y, i32:$z))),
+ (BFI_INT_eg $x, $y, $z)
+>;
+
+// 64-bit version
+def : AMDGPUPat <
+ (xor i64:$z, (and i64:$x, (xor i64:$y, i64:$z))),
+ (REG_SEQUENCE R600_Reg64,
+ (BFI_INT_eg (i32 (EXTRACT_SUBREG R600_Reg64:$x, sub0)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$y, sub0)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$z, sub0))), sub0,
+ (BFI_INT_eg (i32 (EXTRACT_SUBREG R600_Reg64:$x, sub1)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$y, sub1)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$z, sub1))), sub1)
+>;
+
+def : AMDGPUPat <
+ (fcopysign f32:$src0, f32:$src1),
+ (BFI_INT_eg (MOV_IMM_I32 (i32 0x7fffffff)), $src0, $src1)
+>;
+
+def : AMDGPUPat <
+ (fcopysign f32:$src0, f64:$src1),
+ (BFI_INT_eg (MOV_IMM_I32 (i32 0x7fffffff)), $src0,
+ (i32 (EXTRACT_SUBREG R600_Reg64:$src1, sub1)))
+>;
+
+def : AMDGPUPat <
+ (fcopysign f64:$src0, f64:$src1),
+ (REG_SEQUENCE R600_Reg64,
+ (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
+ (BFI_INT_eg (MOV_IMM_I32 (i32 0x7fffffff)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$src0, sub1)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$src1, sub1))), sub1)
+>;
+
+def : AMDGPUPat <
+ (fcopysign f64:$src0, f32:$src1),
+ (REG_SEQUENCE R600_Reg64,
+ (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
+ (BFI_INT_eg (MOV_IMM_I32 (i32 0x7fffffff)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$src0, sub1)),
+ $src1), sub1)
+>;
def BFM_INT_eg : R600_2OP <0xA0, "BFM_INT",
[(set i32:$dst, (AMDGPUbfm i32:$src0, i32:$src1))],
@@ -692,8 +759,26 @@ def : EGOrCaymanPat<(fp_to_sint f32:$src0), (FLT_TO_INT_eg (TRUNC $src0))>;
def : EGOrCaymanPat<(fp_to_uint f32:$src0), (FLT_TO_UINT_eg (TRUNC $src0))>;
-// SHA-256 Patterns
-defm : SHA256MaPattern <BFI_INT_eg, XOR_INT, R600_Reg64>;
+// SHA-256 Ma patterns
+
+// ((x & z) | (y & (x | z))) -> BFI (XOR x, y), z, y
+def : AMDGPUPat <
+ (or (and i32:$x, i32:$z), (and i32:$y, (or i32:$x, i32:$z))),
+ (BFI_INT_eg (XOR_INT i32:$x, i32:$y), i32:$z, i32:$y)
+>;
+
+def : AMDGPUPat <
+ (or (and i64:$x, i64:$z), (and i64:$y, (or i64:$x, i64:$z))),
+ (REG_SEQUENCE R600_Reg64,
+ (BFI_INT_eg (XOR_INT (i32 (EXTRACT_SUBREG R600_Reg64:$x, sub0)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$y, sub0))),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$z, sub0)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$y, sub0))), sub0,
+ (BFI_INT_eg (XOR_INT (i32 (EXTRACT_SUBREG R600_Reg64:$x, sub1)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$y, sub1))),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$z, sub1)),
+ (i32 (EXTRACT_SUBREG R600_Reg64:$y, sub1))), sub1)
+>;
def EG_ExportSwz : ExportSwzInst {
let Word1{19-16} = 0; // BURST_COUNT
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 24f92d5bc522..ab4e06977946 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1550,8 +1550,76 @@ def : GCNPat <
def : IMad24Pat<V_MAD_I32_I24, 1>;
def : UMad24Pat<V_MAD_U32_U24, 1>;
+// BFI patterns
// FIXME: This should only be done for VALU inputs
-defm : BFIPatterns <V_BFI_B32, S_MOV_B32, SReg_64>;
+
+// Definition from ISA doc:
+// (y & x) | (z & ~x)
+def : AMDGPUPat <
+ (or (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))),
+ (V_BFI_B32 $x, $y, $z)
+>;
+
+// 64-bit version
+def : AMDGPUPat <
+ (or (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))),
+ (REG_SEQUENCE SReg_64,
+ (V_BFI_B32 (i32 (EXTRACT_SUBREG SReg_64:$x, sub0)),
+ (i32 (EXTRACT_SUBREG SReg_64:$y, sub0)),
+ (i32 (EXTRACT_SUBREG SReg_64:$z, sub0))), sub0,
+ (V_BFI_B32 (i32 (EXTRACT_SUBREG SReg_64:$x, sub1)),
+ (i32 (EXTRACT_SUBREG SReg_64:$y, sub1)),
+ (i32 (EXTRACT_SUBREG SReg_64:$z, sub1))), sub1)
+>;
+
+// SHA-256 Ch function
+// z ^ (x & (y ^ z))
+def : AMDGPUPat <
+ (xor i32:$z, (and i32:$x, (xor i32:$y, i32:$z))),
+ (V_BFI_B32 $x, $y, $z)
+>;
+
+// 64-bit version
+def : AMDGPUPat <
+ (xor i64:$z, (and i64:$x, (xor i64:$y, i64:$z))),
+ (REG_SEQUENCE SReg_64,
+ (V_BFI_B32 (i32 (EXTRACT_SUBREG SReg_64:$x, sub0)),
+ (i32 (EXTRACT_SUBREG SReg_64:$y, sub0)),
+ (i32 (EXTRACT_SUBREG SReg_64:$z, sub0))), sub0,
+ (V_BFI_B32 (i32 (EXTRACT_SUBREG SReg_64:$x, sub1)),
+ (i32 (EXTRACT_SUBREG SReg_64:$y, sub1)),
+ (i32 (EXTRACT_SUBREG SReg_64:$z, sub1))), sub1)
+>;
+
+def : AMDGPUPat <
+ (fcopysign f32:$src0, f32:$src1),
+ (V_BFI_B32 (S_MOV_B32 (i32 0x7fffffff)), $src0, $src1)
+>;
+
+def : AMDGPUPat <
+ (fcopysign f32:$src0, f64:$src1),
+ (V_BFI_B32 (S_MOV_B32 (i32 0x7fffffff)), $src0,
+ (i32 (EXTRACT_SUBREG SReg_64:$src1, sub1)))
+>;
+
+def : AMDGPUPat <
+ (fcopysign f64:$src0, f64:$src1),
+ (REG_SEQUENCE SReg_64,
+ (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
+ (V_BFI_B32 (S_MOV_B32 (i32 0x7fffffff)),
+ (i32 (EXTRACT_SUBREG SReg_64:$src0, sub1)),
+ (i32 (EXTRACT_SUBREG SReg_64:$src1, sub1))), sub1)
+>;
+
+def : AMDGPUPat <
+ (fcopysign f64:$src0, f32:$src1),
+ (REG_SEQUENCE SReg_64,
+ (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
+ (V_BFI_B32 (S_MOV_B32 (i32 0x7fffffff)),
+ (i32 (EXTRACT_SUBREG SReg_64:$src0, sub1)),
+ $src1), sub1)
+>;
+
def : ROTRPattern <V_ALIGNBIT_B32>;
def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
@@ -2232,7 +2300,27 @@ defm : BFMPatterns <i32, S_BFM_B32, S_MOV_B32>;
// FIXME: defm : BFMPatterns <i64, S_BFM_B64, S_MOV_B64>;
defm : BFEPattern <V_BFE_U32, V_BFE_I32, S_MOV_B32>;
-defm : SHA256MaPattern <V_BFI_B32, V_XOR_B32_e64, SReg_64>;
+
+// SHA-256 Ma patterns
+
+// ((x & z) | (y & (x | z))) -> BFI (XOR x, y), z, y
+def : AMDGPUPat <
+ (or (and i32:$x, i32:$z), (and i32:$y, (or i32:$x, i32:$z))),
+ (V_BFI_B32 (V_XOR_B32_e64 i32:$x, i32:$y), i32:$z, i32:$y)
+>;
+
+def : AMDGPUPat <
+ (or (and i64:$x, i64:$z), (and i64:$y, (or i64:$x, i64:$z))),
+ (REG_SEQUENCE SReg_64,
+ (V_BFI_B32 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG SReg_64:$x, sub0)),
+ (i32 (EXTRACT_SUBREG SReg_64:$y, sub0))),
+ (i32 (EXTRACT_SUBREG SReg_64:$z, sub0)),
+ (i32 (EXTRACT_SUBREG SReg_64:$y, sub0))), sub0,
+ (V_BFI_B32 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG SReg_64:$x, sub1)),
+ (i32 (EXTRACT_SUBREG SReg_64:$y, sub1))),
+ (i32 (EXTRACT_SUBREG SReg_64:$z, sub1)),
+ (i32 (EXTRACT_SUBREG SReg_64:$y, sub1))), sub1)
+>;
multiclass IntMed3Pat<Instruction med3Inst,
SDPatternOperator min,
More information about the llvm-commits
mailing list