[llvm] r312297 - AMDGPU: Fold clamp modifier for packed instructions
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 31 16:53:50 PDT 2017
Author: arsenm
Date: Thu Aug 31 16:53:50 2017
New Revision: 312297
URL: http://llvm.org/viewvc/llvm-project?rev=312297&view=rev
Log:
AMDGPU: Fold clamp modifier for packed instructions
Modified:
llvm/trunk/lib/Target/AMDGPU/SIDefines.h
llvm/trunk/lib/Target/AMDGPU/SIFoldOperands.cpp
llvm/trunk/lib/Target/AMDGPU/SIInstrFormats.td
llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h
llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td
llvm/trunk/lib/Target/AMDGPU/VOPInstructions.td
llvm/trunk/test/CodeGen/AMDGPU/clamp-modifier.ll
Modified: llvm/trunk/lib/Target/AMDGPU/SIDefines.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/SIDefines.h?rev=312297&r1=312296&r2=312297&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIDefines.h (original)
+++ llvm/trunk/lib/Target/AMDGPU/SIDefines.h Thu Aug 31 16:53:50 2017
@@ -67,11 +67,22 @@ enum : uint64_t {
SCALAR_STORE = UINT64_C(1) << 39,
FIXED_SIZE = UINT64_C(1) << 40,
VOPAsmPrefer32Bit = UINT64_C(1) << 41,
- HasFPClamp = UINT64_C(1) << 42,
- VOP3_OPSEL = UINT64_C(1) << 43,
- maybeAtomic = UINT64_C(1) << 44,
- F16_ZFILL = UINT64_C(1) << 45,
- IntClamp = UINT64_C(1) << 46
+ VOP3_OPSEL = UINT64_C(1) << 42,
+ maybeAtomic = UINT64_C(1) << 43,
+ F16_ZFILL = UINT64_C(1) << 44,
+
+ // Is a clamp on FP type.
+ FPClamp = UINT64_C(1) << 45,
+
+ // Is an integer clamp
+ IntClamp = UINT64_C(1) << 46,
+
+ // Clamps lo component of register.
+ ClampLo = UINT64_C(1) << 47,
+
+ // Clamps hi component of register.
+ // ClampLo and ClampHi set for packed clamp.
+ ClampHi = UINT64_C(1) << 48
};
// v_cmp_class_* etc. use a 10-bit mask for what operation is checked.
Modified: llvm/trunk/lib/Target/AMDGPU/SIFoldOperands.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/SIFoldOperands.cpp?rev=312297&r1=312296&r2=312297&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIFoldOperands.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/SIFoldOperands.cpp Thu Aug 31 16:53:50 2017
@@ -728,7 +728,8 @@ const MachineOperand *SIFoldOperands::is
switch (Op) {
case AMDGPU::V_MAX_F32_e64:
case AMDGPU::V_MAX_F16_e64:
- case AMDGPU::V_MAX_F64: {
+ case AMDGPU::V_MAX_F64:
+ case AMDGPU::V_PK_MAX_F16: {
if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm())
return nullptr;
@@ -741,9 +742,18 @@ const MachineOperand *SIFoldOperands::is
return nullptr;
// Can't fold up if we have modifiers.
- if (TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
- TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
- TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
+ if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
+ return nullptr;
+
+ unsigned Src0Mods
+ = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm();
+ unsigned Src1Mods
+ = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm();
+
+ // Having a 0 op_sel_hi would require swizzling the output in the source
+ // instruction, which we can't do.
+ unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1 : 0;
+ if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
return nullptr;
return Src0;
}
@@ -771,8 +781,11 @@ bool SIFoldOperands::tryFoldClamp(Machin
return false;
MachineInstr *Def = MRI->getVRegDef(ClampSrc->getReg());
- if (!TII->hasFPClamp(*Def))
+
+ // The type of clamp must be compatible.
+ if (TII->getClampMask(*Def) != TII->getClampMask(MI))
return false;
+
MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp);
if (!DefClamp)
return false;
Modified: llvm/trunk/lib/Target/AMDGPU/SIInstrFormats.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/SIInstrFormats.td?rev=312297&r1=312296&r2=312297&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIInstrFormats.td (original)
+++ llvm/trunk/lib/Target/AMDGPU/SIInstrFormats.td Thu Aug 31 16:53:50 2017
@@ -79,10 +79,6 @@ class InstSI <dag outs, dag ins, string
// is unable to infer the encoding from the operands.
field bit VOPAsmPrefer32Bit = 0;
- // This bit indicates that this has a floating point result type, so
- // the clamp modifier has floating point semantics.
- field bit FPClamp = 0;
-
// This bit indicates that this is a VOP3 opcode which supports op_sel
// modifier (gfx9 only).
field bit VOP3_OPSEL = 0;
@@ -94,10 +90,22 @@ class InstSI <dag outs, dag ins, string
// unused bits in dst. Note that new GFX9 opcodes preserve unused bits.
field bit F16_ZFILL = 0;
+ // This bit indicates that this has a floating point result type, so
+ // the clamp modifier has floating point semantics.
+ field bit FPClamp = 0;
+
// This bit indicates that instruction may support integer clamping
// which depends on GPU features.
field bit IntClamp = 0;
+ // This field indicates that the clamp applies to the low component
+ // of a packed output register.
+ field bit ClampLo = 0;
+
+ // This field indicates that the clamp applies to the high component
+ // of a packed output register.
+ field bit ClampHi = 0;
+
// These need to be kept in sync with the enum in SIInstrFlags.
let TSFlags{0} = SALU;
let TSFlags{1} = VALU;
@@ -141,12 +149,15 @@ class InstSI <dag outs, dag ins, string
let TSFlags{39} = ScalarStore;
let TSFlags{40} = FixedSize;
let TSFlags{41} = VOPAsmPrefer32Bit;
- let TSFlags{42} = FPClamp;
- let TSFlags{43} = VOP3_OPSEL;
+ let TSFlags{42} = VOP3_OPSEL;
+
+ let TSFlags{43} = maybeAtomic;
+ let TSFlags{44} = F16_ZFILL;
- let TSFlags{44} = maybeAtomic;
- let TSFlags{45} = F16_ZFILL;
+ let TSFlags{45} = FPClamp;
let TSFlags{46} = IntClamp;
+ let TSFlags{47} = ClampLo;
+ let TSFlags{48} = ClampHi;
let SchedRW = [Write32Bit];
Modified: llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h?rev=312297&r1=312296&r2=312297&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h (original)
+++ llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h Thu Aug 31 16:53:50 2017
@@ -548,11 +548,23 @@ public:
}
static bool hasFPClamp(const MachineInstr &MI) {
- return MI.getDesc().TSFlags & SIInstrFlags::HasFPClamp;
+ return MI.getDesc().TSFlags & SIInstrFlags::FPClamp;
}
bool hasFPClamp(uint16_t Opcode) const {
- return get(Opcode).TSFlags & SIInstrFlags::HasFPClamp;
+ return get(Opcode).TSFlags & SIInstrFlags::FPClamp;
+ }
+
+ static bool hasIntClamp(const MachineInstr &MI) {
+ return MI.getDesc().TSFlags & SIInstrFlags::IntClamp;
+ }
+
+ uint64_t getClampMask(const MachineInstr &MI) const {
+ const uint64_t ClampFlags = SIInstrFlags::FPClamp |
+ SIInstrFlags::IntClamp |
+ SIInstrFlags::ClampLo |
+ SIInstrFlags::ClampHi;
+ return MI.getDesc().TSFlags & ClampFlags;
}
bool isVGPRCopy(const MachineInstr &MI) const {
Modified: llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td?rev=312297&r1=312296&r2=312297&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td (original)
+++ llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td Thu Aug 31 16:53:50 2017
@@ -1529,6 +1529,8 @@ class VOPProfile <list<ValueType> _ArgVT
field bit HasSDWAClamp = EmitDst;
field bit HasFPClamp = BitAnd<isFloatType<DstVT>.ret, HasClamp>.ret;
field bit HasIntClamp = !if(isFloatType<DstVT>.ret, 0, HasClamp);
+ field bit HasClampLo = HasClamp;
+ field bit HasClampHi = BitAnd<isPackedType<DstVT>.ret, HasClamp>.ret;
field bit HasHigh = 0;
field bit IsPacked = isPackedType<Src0VT>.ret;
Modified: llvm/trunk/lib/Target/AMDGPU/VOPInstructions.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/VOPInstructions.td?rev=312297&r1=312296&r2=312297&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/VOPInstructions.td (original)
+++ llvm/trunk/lib/Target/AMDGPU/VOPInstructions.td Thu Aug 31 16:53:50 2017
@@ -106,6 +106,10 @@ class VOP3_Pseudo <string opName, VOPPro
let VOP3 = 1;
let VALU = 1;
let FPClamp = P.HasFPClamp;
+ let IntClamp = P.HasIntClamp;
+ let ClampLo = P.HasClampLo;
+ let ClampHi = P.HasClampHi;
+
let Uses = [EXEC];
let AsmVariantName = AMDGPUAsmVariants.VOP3;
Modified: llvm/trunk/test/CodeGen/AMDGPU/clamp-modifier.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/clamp-modifier.ll?rev=312297&r1=312296&r2=312297&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/AMDGPU/clamp-modifier.ll (original)
+++ llvm/trunk/test/CodeGen/AMDGPU/clamp-modifier.ll Thu Aug 31 16:53:50 2017
@@ -1,8 +1,9 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9 %s
; GCN-LABEL: {{^}}v_clamp_add_src_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
; GCN-NOT: [[A]]
; GCN: v_add_f32_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
define amdgpu_kernel void @v_clamp_add_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -18,7 +19,7 @@ define amdgpu_kernel void @v_clamp_add_s
}
; GCN-LABEL: {{^}}v_clamp_multi_use_src_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
; GCN: v_max_f32_e64 v{{[0-9]+}}, [[ADD]], [[ADD]] clamp{{$}}
define amdgpu_kernel void @v_clamp_multi_use_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -35,7 +36,7 @@ define amdgpu_kernel void @v_clamp_multi
}
; GCN-LABEL: {{^}}v_clamp_dbg_use_src_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
; GCN-NOT: [[A]]
; GCN: v_add_f32_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
define amdgpu_kernel void @v_clamp_dbg_use_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -52,7 +53,7 @@ define amdgpu_kernel void @v_clamp_dbg_u
}
; GCN-LABEL: {{^}}v_clamp_add_neg_src_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
; GCN: v_floor_f32_e32 [[FLOOR:v[0-9]+]], [[A]]
; GCN: v_max_f32_e64 v{{[0-9]+}}, -[[FLOOR]], -[[FLOOR]] clamp{{$}}
define amdgpu_kernel void @v_clamp_add_neg_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -69,7 +70,7 @@ define amdgpu_kernel void @v_clamp_add_n
}
; GCN-LABEL: {{^}}v_non_clamp_max_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
; GCN: v_max_f32_e32 v{{[0-9]+}}, 0, [[ADD]]{{$}}
define amdgpu_kernel void @v_non_clamp_max_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -84,7 +85,7 @@ define amdgpu_kernel void @v_non_clamp_m
}
; GCN-LABEL: {{^}}v_clamp_add_src_f32_denormals:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
; GCN: v_add_f32_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
define amdgpu_kernel void @v_clamp_add_src_f32_denormals(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -99,8 +100,8 @@ define amdgpu_kernel void @v_clamp_add_s
}
; GCN-LABEL: {{^}}v_clamp_add_src_f16_denorm:
-; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
-; VI: v_add_f16_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
+; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX89: v_add_f16_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]]
; SI: v_add_f32_e64 [[ADD:v[0-9]+]], [[CVT]], 1.0 clamp{{$}}
@@ -118,9 +119,9 @@ define amdgpu_kernel void @v_clamp_add_s
}
; GCN-LABEL: {{^}}v_clamp_add_src_f16_no_denormals:
-; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
-; VI-NOT: [[A]]
-; VI: v_add_f16_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
+; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX89-NOT: [[A]]
+; GFX89: v_add_f16_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]]
; SI: v_add_f32_e64 [[ADD:v[0-9]+]], [[CVT]], 1.0 clamp{{$}}
@@ -138,7 +139,7 @@ define amdgpu_kernel void @v_clamp_add_s
}
; GCN-LABEL: {{^}}v_clamp_add_src_v2f32:
-; GCN: {{buffer|flat}}_load_dwordx2 v{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}}
+; GCN: {{buffer|flat|global}}_load_dwordx2 v{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}}
; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, v[[A]], 1.0 clamp{{$}}
; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, v[[B]], 1.0 clamp{{$}}
define amdgpu_kernel void @v_clamp_add_src_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %aptr) #0 {
@@ -154,7 +155,7 @@ define amdgpu_kernel void @v_clamp_add_s
}
; GCN-LABEL: {{^}}v_clamp_add_src_f64:
-; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
+; GCN: {{buffer|flat|global}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, [[A]], 1.0 clamp{{$}}
define amdgpu_kernel void @v_clamp_add_src_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -185,6 +186,173 @@ define amdgpu_kernel void @v_clamp_mac_t
ret void
}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+ %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+ %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+ %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %add, <2 x half> zeroinitializer)
+ %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+ store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+ ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_no_denormals:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_no_denormals(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #3 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+ %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+ %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+ %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %add, <2 x half> zeroinitializer)
+ %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+ store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+ ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_lo:[1,1] neg_hi:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+ %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+ %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+ %neg.add = fsub <2 x half> <half -0.0, half -0.0>, %add
+ %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.add, <2 x half> zeroinitializer)
+ %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+ store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+ ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg_lo:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_lo:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg_lo(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+ %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+ %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+ %lo = extractelement <2 x half> %add, i32 0
+ %neg.lo = fsub half -0.0, %lo
+ %neg.lo.add = insertelement <2 x half> %add, half %neg.lo, i32 0
+ %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.lo.add, <2 x half> zeroinitializer)
+ %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+ store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+ ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg_hi:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_hi:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg_hi(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+ %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+ %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+ %hi = extractelement <2 x half> %add, i32 1
+ %neg.hi = fsub half -0.0, %hi
+ %neg.hi.add = insertelement <2 x half> %add, half %neg.hi, i32 1
+ %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.hi.add, <2 x half> zeroinitializer)
+ %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+ store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+ ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_shuf:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] op_sel:[1,1] op_sel_hi:[0,0] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_shuf(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+ %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+ %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+ %shuf = shufflevector <2 x half> %add, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+
+ %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %shuf, <2 x half> zeroinitializer)
+ %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+ store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+ ret void
+}
+
+; GCN-LABEL: {{^}}v_no_clamp_add_src_v2f16_f32_src:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
+define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f32_src(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+ %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+ %bc = bitcast <2 x half> %a to float
+ %f32.op = fadd float %bc, 1.0
+ %f32.op.cast = bitcast float %f32.op to <2 x half>
+ %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %f32.op.cast, <2 x half> zeroinitializer)
+ %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+ store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+ ret void
+}
+
+; GCN-LABEL: {{^}}v_no_clamp_add_packed_src_f32:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_max_f32_e64 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
+define amdgpu_kernel void @v_no_clamp_add_packed_src_f32(float addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
+ %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+ %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+ %bc.add = bitcast <2 x half> %add to float
+ %max = call float @llvm.maxnum.f32(float %bc.add, float 0.0)
+ %clamp = call float @llvm.minnum.f32(float %max, float 1.0)
+ store float %clamp, float addrspace(1)* %out.gep
+ ret void
+}
+
+; Since the high bits are zeroed, it probably would be OK in this case
+; to use clamp.
+; GCN-LABEL: {{^}}v_no_clamp_add_src_v2f16_f16_src:
+; GCN-DAG: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX9: v_add_f16_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
+define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f16_src(<2 x half> addrspace(1)* %out, half addrspace(1)* %aptr) #0 {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep0 = getelementptr half, half addrspace(1)* %aptr, i32 %tid
+ %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+ %a = load half, half addrspace(1)* %gep0
+ %add = fadd half %a, 1.0
+ %bc = bitcast half %add to i16
+ %zext = zext i16 %bc to i32
+ %v2f16 = bitcast i32 %zext to <2 x half>
+ %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %v2f16, <2 x half> zeroinitializer)
+ %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+ store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+ ret void
+}
+
declare i32 @llvm.amdgcn.workitem.id.x() #1
declare float @llvm.fabs.f32(float) #1
declare float @llvm.floor.f32(float) #1
@@ -197,8 +365,12 @@ declare double @llvm.maxnum.f64(double,
declare half @llvm.fabs.f16(half) #1
declare half @llvm.minnum.f16(half, half) #1
declare half @llvm.maxnum.f16(half, half) #1
+declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>) #1
+declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>) #1
declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) #1
declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>) #1
+
+
declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
attributes #0 = { nounwind }
More information about the llvm-commits
mailing list