[llvm] r216735 - R600/SI: Use mad for fsub + fmul
Matt Arsenault
Matthew.Arsenault at amd.com
Fri Aug 29 09:01:15 PDT 2014
Author: arsenm
Date: Fri Aug 29 11:01:14 2014
New Revision: 216735
URL: http://llvm.org/viewvc/llvm-project?rev=216735&view=rev
Log:
R600/SI: Use mad for fsub + fmul
We can use a negate source modifier to match
this for fsub.
Added:
llvm/trunk/test/CodeGen/R600/mad-sub.ll
Modified:
llvm/trunk/lib/Target/R600/AMDGPUISelLowering.cpp
llvm/trunk/lib/Target/R600/AMDGPUISelLowering.h
llvm/trunk/lib/Target/R600/AMDGPUInstrInfo.td
llvm/trunk/lib/Target/R600/SIISelLowering.cpp
llvm/trunk/lib/Target/R600/SIInstrInfo.td
llvm/trunk/lib/Target/R600/SIInstructions.td
Modified: llvm/trunk/lib/Target/R600/AMDGPUISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/AMDGPUISelLowering.cpp?rev=216735&r1=216734&r2=216735&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/AMDGPUISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/R600/AMDGPUISelLowering.cpp Fri Aug 29 11:01:14 2014
@@ -2157,6 +2157,7 @@ const char* AMDGPUTargetLowering::getTar
NODE_NAME_CASE(DWORDADDR)
NODE_NAME_CASE(FRACT)
NODE_NAME_CASE(CLAMP)
+ NODE_NAME_CASE(MAD)
NODE_NAME_CASE(FMAX)
NODE_NAME_CASE(SMAX)
NODE_NAME_CASE(UMAX)
Modified: llvm/trunk/lib/Target/R600/AMDGPUISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/AMDGPUISelLowering.h?rev=216735&r1=216734&r2=216735&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/AMDGPUISelLowering.h (original)
+++ llvm/trunk/lib/Target/R600/AMDGPUISelLowering.h Fri Aug 29 11:01:14 2014
@@ -180,6 +180,7 @@ enum {
DWORDADDR,
FRACT,
CLAMP,
+ MAD, // Multiply + add with same result as the separate operations.
// SIN_HW, COS_HW - f32 for SI, 1 ULP max error, valid from -100 pi to 100 pi.
// Denormals handled on some parts.
Modified: llvm/trunk/lib/Target/R600/AMDGPUInstrInfo.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/AMDGPUInstrInfo.td?rev=216735&r1=216734&r2=216735&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/AMDGPUInstrInfo.td (original)
+++ llvm/trunk/lib/Target/R600/AMDGPUInstrInfo.td Fri Aug 29 11:01:14 2014
@@ -64,6 +64,7 @@ def AMDGPUfmax : SDNode<"AMDGPUISD::FMAX
>;
def AMDGPUclamp : SDNode<"AMDGPUISD::CLAMP", SDTFPTernaryOp, []>;
+def AMDGPUmad : SDNode<"AMDGPUISD::MAD", SDTFPTernaryOp, []>;
// out = max(a, b) a and b are signed ints
def AMDGPUsmax : SDNode<"AMDGPUISD::SMAX", SDTIntBinOp,
Modified: llvm/trunk/lib/Target/R600/SIISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/SIISelLowering.cpp?rev=216735&r1=216734&r2=216735&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/SIISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/R600/SIISelLowering.cpp Fri Aug 29 11:01:14 2014
@@ -226,6 +226,7 @@ SITargetLowering::SITargetLowering(Targe
setOperationAction(ISD::FDIV, MVT::f32, Custom);
+ setTargetDAGCombine(ISD::FSUB);
setTargetDAGCombine(ISD::SELECT_CC);
setTargetDAGCombine(ISD::SETCC);
@@ -1386,6 +1387,42 @@ SDValue SITargetLowering::PerformDAGComb
case ISD::UINT_TO_FP: {
return performUCharToFloatCombine(N, DCI);
+
+ case ISD::FSUB: {
+ if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
+ break;
+
+ EVT VT = N->getValueType(0);
+
+ // Try to get the fneg to fold into the source modifier. This undoes generic
+ // DAG combines and folds them into the mad.
+ if (VT == MVT::f32) {
+ SDValue LHS = N->getOperand(0);
+ SDValue RHS = N->getOperand(1);
+
+ if (LHS.getOpcode() == ISD::FMUL) {
+ // (fsub (fmul a, b), c) -> mad a, b, (fneg c)
+
+ SDValue A = LHS.getOperand(0);
+ SDValue B = LHS.getOperand(1);
+ SDValue C = DAG.getNode(ISD::FNEG, DL, VT, RHS);
+
+ return DAG.getNode(AMDGPUISD::MAD, DL, VT, A, B, C);
+ }
+
+ if (RHS.getOpcode() == ISD::FMUL) {
+ // (fsub c, (fmul a, b)) -> mad (fneg a), b, c
+
+ SDValue A = DAG.getNode(ISD::FNEG, DL, VT, RHS.getOperand(0));
+ SDValue B = RHS.getOperand(1);
+ SDValue C = LHS;
+
+ return DAG.getNode(AMDGPUISD::MAD, DL, VT, A, B, C);
+ }
+ }
+
+ break;
+ }
}
case ISD::LOAD:
case ISD::STORE:
Modified: llvm/trunk/lib/Target/R600/SIInstrInfo.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/SIInstrInfo.td?rev=216735&r1=216734&r2=216735&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/SIInstrInfo.td (original)
+++ llvm/trunk/lib/Target/R600/SIInstrInfo.td Fri Aug 29 11:01:14 2014
@@ -796,6 +796,17 @@ multiclass VOP3b_64 <bits<9> op, string
multiclass VOP3b_32 <bits<9> op, string opName, list<dag> pattern> :
VOP3b_Helper <op, VReg_32, VSrc_32, opName, pattern>;
+
+class Vop3ModPat<Instruction Inst, VOPProfile P, SDPatternOperator node> : Pat<
+ (node (P.Src0VT (VOP3Mods0 P.Src0VT:$src0, i32:$src0_modifiers, i32:$clamp, i32:$omod)),
+ (P.Src1VT (VOP3Mods P.Src1VT:$src1, i32:$src1_modifiers)),
+ (P.Src2VT (VOP3Mods P.Src2VT:$src2, i32:$src2_modifiers))),
+ (Inst i32:$src0_modifiers, P.Src0VT:$src0,
+ i32:$src1_modifiers, P.Src1VT:$src1,
+ i32:$src2_modifiers, P.Src2VT:$src2,
+ i32:$clamp,
+ i32:$omod)>;
+
//===----------------------------------------------------------------------===//
// Vector I/O classes
//===----------------------------------------------------------------------===//
Modified: llvm/trunk/lib/Target/R600/SIInstructions.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/SIInstructions.td?rev=216735&r1=216734&r2=216735&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/SIInstructions.td (original)
+++ llvm/trunk/lib/Target/R600/SIInstructions.td Fri Aug 29 11:01:14 2014
@@ -2513,6 +2513,9 @@ def : Pat <
(V_MUL_HI_I32 $src0, $src1)
>;
+def : Vop3ModPat<V_MAD_F32, VOP_F32_F32_F32_F32, AMDGPUmad>;
+
+
defm : BFIPatterns <V_BFI_B32, S_MOV_B32>;
def : ROTRPattern <V_ALIGNBIT_B32>;
Added: llvm/trunk/test/CodeGen/R600/mad-sub.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/R600/mad-sub.ll?rev=216735&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/R600/mad-sub.ll (added)
+++ llvm/trunk/test/CodeGen/R600/mad-sub.ll Fri Aug 29 11:01:14 2014
@@ -0,0 +1,173 @@
+; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+
+declare i32 @llvm.r600.read.tidig.x() #0
+declare float @llvm.fabs.f32(float) #0
+
+; FUNC-LABEL: @mad_sub_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -[[REGC]]
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_sub_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+ %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+ %tid.ext = sext i32 %tid to i64
+ %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+ %add1 = add i64 %tid.ext, 1
+ %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+ %add2 = add i64 %tid.ext, 2
+ %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+ %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+ %a = load float addrspace(1)* %gep0, align 4
+ %b = load float addrspace(1)* %gep1, align 4
+ %c = load float addrspace(1)* %gep2, align 4
+ %mul = fmul float %a, %b
+ %sub = fsub float %mul, %c
+ store float %sub, float addrspace(1)* %outgep, align 4
+ ret void
+}
+
+; FUNC-LABEL: @mad_sub_inv_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], [[REGC]]
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_sub_inv_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+ %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+ %tid.ext = sext i32 %tid to i64
+ %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+ %add1 = add i64 %tid.ext, 1
+ %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+ %add2 = add i64 %tid.ext, 2
+ %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+ %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+ %a = load float addrspace(1)* %gep0, align 4
+ %b = load float addrspace(1)* %gep1, align 4
+ %c = load float addrspace(1)* %gep2, align 4
+ %mul = fmul float %a, %b
+ %sub = fsub float %c, %mul
+ store float %sub, float addrspace(1)* %outgep, align 4
+ ret void
+}
+
+; FUNC-LABEL: @mad_sub_f64
+; SI: V_MUL_F64
+; SI: V_ADD_F64
+define void @mad_sub_f64(double addrspace(1)* noalias nocapture %out, double addrspace(1)* noalias nocapture readonly %ptr) #1 {
+ %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+ %tid.ext = sext i32 %tid to i64
+ %gep0 = getelementptr double addrspace(1)* %ptr, i64 %tid.ext
+ %add1 = add i64 %tid.ext, 1
+ %gep1 = getelementptr double addrspace(1)* %ptr, i64 %add1
+ %add2 = add i64 %tid.ext, 2
+ %gep2 = getelementptr double addrspace(1)* %ptr, i64 %add2
+ %outgep = getelementptr double addrspace(1)* %out, i64 %tid.ext
+ %a = load double addrspace(1)* %gep0, align 8
+ %b = load double addrspace(1)* %gep1, align 8
+ %c = load double addrspace(1)* %gep2, align 8
+ %mul = fmul double %a, %b
+ %sub = fsub double %mul, %c
+ store double %sub, double addrspace(1)* %outgep, align 8
+ ret void
+}
+
+; FUNC-LABEL: @mad_sub_fabs_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -|[[REGC]]|
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_sub_fabs_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+ %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+ %tid.ext = sext i32 %tid to i64
+ %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+ %add1 = add i64 %tid.ext, 1
+ %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+ %add2 = add i64 %tid.ext, 2
+ %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+ %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+ %a = load float addrspace(1)* %gep0, align 4
+ %b = load float addrspace(1)* %gep1, align 4
+ %c = load float addrspace(1)* %gep2, align 4
+ %c.abs = call float @llvm.fabs.f32(float %c) #0
+ %mul = fmul float %a, %b
+ %sub = fsub float %mul, %c.abs
+ store float %sub, float addrspace(1)* %outgep, align 4
+ ret void
+}
+
+; FUNC-LABEL: @mad_sub_fabs_inv_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], |[[REGC]]|
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_sub_fabs_inv_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+ %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+ %tid.ext = sext i32 %tid to i64
+ %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+ %add1 = add i64 %tid.ext, 1
+ %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+ %add2 = add i64 %tid.ext, 2
+ %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+ %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+ %a = load float addrspace(1)* %gep0, align 4
+ %b = load float addrspace(1)* %gep1, align 4
+ %c = load float addrspace(1)* %gep2, align 4
+ %c.abs = call float @llvm.fabs.f32(float %c) #0
+ %mul = fmul float %a, %b
+ %sub = fsub float %c.abs, %mul
+ store float %sub, float addrspace(1)* %outgep, align 4
+ ret void
+}
+
+; FUNC-LABEL: @neg_neg_mad_f32
+; SI: V_MAD_F32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+define void @neg_neg_mad_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+ %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+ %tid.ext = sext i32 %tid to i64
+ %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+ %add1 = add i64 %tid.ext, 1
+ %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+ %add2 = add i64 %tid.ext, 2
+ %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+ %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+ %a = load float addrspace(1)* %gep0, align 4
+ %b = load float addrspace(1)* %gep1, align 4
+ %c = load float addrspace(1)* %gep2, align 4
+ %nega = fsub float -0.000000e+00, %a
+ %negb = fsub float -0.000000e+00, %b
+ %mul = fmul float %nega, %negb
+ %sub = fadd float %mul, %c
+ store float %sub, float addrspace(1)* %outgep, align 4
+ ret void
+}
+
+; FUNC-LABEL: @mad_fabs_sub_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], [[REGA]], |[[REGB]]|, -[[REGC]]
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_fabs_sub_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+ %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+ %tid.ext = sext i32 %tid to i64
+ %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+ %add1 = add i64 %tid.ext, 1
+ %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+ %add2 = add i64 %tid.ext, 2
+ %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+ %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+ %a = load float addrspace(1)* %gep0, align 4
+ %b = load float addrspace(1)* %gep1, align 4
+ %c = load float addrspace(1)* %gep2, align 4
+ %b.abs = call float @llvm.fabs.f32(float %b) #0
+ %mul = fmul float %a, %b.abs
+ %sub = fsub float %mul, %c
+ store float %sub, float addrspace(1)* %outgep, align 4
+ ret void
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
More information about the llvm-commits
mailing list