[llvm] d3ea7f2 - [AMDGPU] mad-combine.ll - regenerate test checks and remove duplicate safe/unsafe RUN line

Thu Jul 10 03:52:20 PDT 2025

Author: Simon Pilgrim
Date: 2025-07-10T11:52:02+01:00
New Revision: d3ea7f29ec50937242ae12c8cd5ce1c3f0667204

URL: https://github.com/llvm/llvm-project/commit/d3ea7f29ec50937242ae12c8cd5ce1c3f0667204
DIFF: https://github.com/llvm/llvm-project/commit/d3ea7f29ec50937242ae12c8cd5ce1c3f0667204.diff

LOG: [AMDGPU] mad-combine.ll - regenerate test checks and remove duplicate safe/unsafe RUN line

Added: 
    

Modified: 
    llvm/test/CodeGen/AMDGPU/mad-combine.ll

Removed: 
    


################################################################################
diff  --git a/llvm/test/CodeGen/AMDGPU/mad-combine.ll b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
index 7b8aa86bee2cc..67138aeabd6cc 100644

--- a/llvm/test/CodeGen/AMDGPU/mad-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
@@ -1,11 +1,9 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; Make sure we still form mad even when unsafe math or fp-contract is allowed instead of fma.
-
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
-
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-STD %s
 ; Make sure we don't form mad with denormals
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-DENORM,SI-DENORM-FASTFMAF %s
+; RUN: llc -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-DENORM,SI-DENORM-SLOWFMAF %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare float @llvm.fabs.f32(float) #0
@@ -13,18 +11,67 @@ declare float @llvm.fma.f32(float, float, float) #0
 declare float @llvm.fmuladd.f32(float, float, float) #0
 
 ; (fadd (fmul x, y), z) -> (fma x, y, z)
-; FUNC-LABEL: {{^}}combine_to_mad_f32_0:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-
-; SI-STD: v_mac_f32_e32 [[C]], [[A]], [[B]]
-
-; SI-DENORM-FASTFMAF: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
-
-; SI-DENORM-FASTFMAF: buffer_store_dword [[RESULT]]
-; SI-STD: buffer_store_dword [[C]]
 define amdgpu_kernel void @combine_to_mad_f32_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_f32_0:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mac_f32_e32 v4, v2, v3
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_f32_0:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, v2, v3, v4
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_f32_0:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_add_f32_e32 v2, v2, v4
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -41,22 +88,48 @@ define amdgpu_kernel void @combine_to_mad_f32_0(ptr addrspace(1) noalias %out, p
   store float %fma, ptr addrspace(1) %gep.out
   ret void
 }
-; FUNC-LABEL: {{^}}no_combine_to_mad_f32_0:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-
-; SI-STD: v_mac_f32_e32 [[C]], [[A]], [[B]]
-
-; SI-DENORM-SLOWFMAF-NOT: v_fma
-; SI-DENORM-SLOWFMAF-NOT: v_mad
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]],  [[TMP]], [[C]]
 
-; SI-DENORM-SLOWFMAF: buffer_store_dword [[RESULT]]
-; SI-STD: buffer_store_dword [[C]]
 define amdgpu_kernel void @no_combine_to_mad_f32_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: no_combine_to_mad_f32_0:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mac_f32_e32 v4, v2, v3
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-LABEL: no_combine_to_mad_f32_0:
+; SI-DENORM:       ; %bb.0:
+; SI-DENORM-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-NEXT:    v_add_f32_e32 v2, v2, v4
+; SI-DENORM-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -74,28 +147,85 @@ define amdgpu_kernel void @no_combine_to_mad_f32_0(ptr addrspace(1) noalias %out
 }
 
 ; (fadd (fmul x, y), z) -> (fma x, y, z)
-; FUNC-LABEL: {{^}}combine_to_mad_f32_0_2use:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-
-; SI-STD-DAG: v_mac_f32_e32 [[C]], [[A]], [[B]]
-; SI-STD-DAG: v_mac_f32_e32 [[D]], [[A]], [[B]]
-
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], [[C]]
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], [[D]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
-; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
-
-; SI-DENORM-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DENORM-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
-; SI-STD-DAG: buffer_store_dword [[C]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-STD-DAG: buffer_store_dword [[D]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_fast) #1 {
+; SI-STD-LABEL: combine_to_mad_f32_0_2use:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mac_f32_e32 v4, v2, v3
+; SI-STD-NEXT:    v_mac_f32_e32 v5, v2, v3
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_store_dword v5, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_f32_0_2use:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v4, v2, v3, v4
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, v2, v3, v5
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_f32_0_2use:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_add_f32_e32 v3, v2, v4
+; SI-DENORM-SLOWFMAF-NEXT:    v_add_f32_e32 v2, v2, v5
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -116,25 +246,60 @@ define amdgpu_kernel void @combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %o
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
   ret void
 }
-; FUNC-LABEL: {{^}}no_combine_to_mad_f32_0_2use:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-
-; SI-STD-DAG: v_mac_f32_e32 [[C]], [[A]], [[B]]
-; SI-STD-DAG: v_mac_f32_e32 [[D]], [[A]], [[B]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
-; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
-
-; SI-DENORM-SLOWFMAF-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DENORM-SLOWFMAF-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
-; SI-STD-DAG: buffer_store_dword [[C]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-STD-DAG: buffer_store_dword [[D]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
-; SI: s_endpgm
+
 define amdgpu_kernel void @no_combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_fast) #1 {
+; SI-STD-LABEL: no_combine_to_mad_f32_0_2use:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mac_f32_e32 v4, v2, v3
+; SI-STD-NEXT:    v_mac_f32_e32 v5, v2, v3
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_store_dword v5, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-LABEL: no_combine_to_mad_f32_0_2use:
+; SI-DENORM:       ; %bb.0:
+; SI-DENORM-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-NEXT:    v_add_f32_e32 v3, v2, v4
+; SI-DENORM-NEXT:    v_add_f32_e32 v2, v2, v5
+; SI-DENORM-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -157,20 +322,67 @@ define amdgpu_kernel void @no_combine_to_mad_f32_0_2use(ptr addrspace(1) noalias
 }
 
 ; (fadd x, (fmul y, z)) -> (fma y, z, x)
-; FUNC-LABEL: {{^}}combine_to_mad_f32_1:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-
-; SI-STD: v_mac_f32_e32 [[C]], [[A]], [[B]]
-; SI-DENORM-FASTFMAF: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
-
-; SI-DENORM: buffer_store_dword [[RESULT]]
-; SI-STD: buffer_store_dword [[C]]
 define amdgpu_kernel void @combine_to_mad_f32_1(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_f32_1:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mac_f32_e32 v4, v2, v3
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_f32_1:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, v2, v3, v4
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_f32_1:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_add_f32_e32 v2, v4, v2
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -188,19 +400,67 @@ define amdgpu_kernel void @combine_to_mad_f32_1(ptr addrspace(1) noalias %out, p
 }
 
 ; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_mad_fsub_0_f32:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-
-; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], -[[C]]
-; SI-DENORM-FASTFMAF: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], -[[C]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
-
-; SI: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_fsub_0_f32:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mad_f32 v2, v2, v3, -v4
+; SI-STD-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_fsub_0_f32:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, v2, v3, -v4
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_fsub_0_f32:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v2, v2, v4
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -218,26 +478,85 @@ define amdgpu_kernel void @combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %o
 }
 
 ; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_mad_fsub_0_f32_2use:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-
-; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], -[[C]]
-; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
-
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], -[[C]]
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
-; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
-
-; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_mad_fsub_0_f32_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_fsub_0_f32_2use:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mad_f32 v4, v2, v3, -v4
+; SI-STD-NEXT:    v_mad_f32 v2, v2, v3, -v5
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_fsub_0_f32_2use:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v4, v2, v3, -v4
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, v2, v3, -v5
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_fsub_0_f32_2use:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v2, v4
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v2, v2, v5
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -260,19 +579,67 @@ define amdgpu_kernel void @combine_to_mad_fsub_0_f32_2use(ptr addrspace(1) noali
 }
 
 ; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
-; FUNC-LABEL: {{^}}combine_to_mad_fsub_1_f32:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-
-; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], [[C]]
-; SI-DENORM-FASTFMAF: v_fma_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], [[C]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
-
-; SI: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @combine_to_mad_fsub_1_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_fsub_1_f32:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mad_f32 v2, -v2, v3, v4
+; SI-STD-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_fsub_1_f32:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, -v2, v3, v4
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_fsub_1_f32:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v2, v4, v2
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -290,26 +657,85 @@ define amdgpu_kernel void @combine_to_mad_fsub_1_f32(ptr addrspace(1) noalias %o
 }
 
 ; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
-; FUNC-LABEL: {{^}}combine_to_mad_fsub_1_f32_2use:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-
-; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
-; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
-
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e32 [[RESULT0:v[0-9]+]], [[C]], [[TMP]]
-; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e32 [[RESULT1:v[0-9]+]],  [[D]], [[TMP]]
-
-; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_mad_fsub_1_f32_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_fsub_1_f32_2use:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mad_f32 v4, -v2, v3, v4
+; SI-STD-NEXT:    v_mad_f32 v2, -v2, v3, v5
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_fsub_1_f32_2use:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v4, -v2, v3, v4
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, -v2, v3, v5
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_fsub_1_f32_2use:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v4, v2
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v2, v5, v2
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -332,20 +758,67 @@ define amdgpu_kernel void @combine_to_mad_fsub_1_f32_2use(ptr addrspace(1) noali
 }
 
 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-
-; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], -[[B]], -[[C]]
-
-; SI-DENORM-FASTFMAF: v_fma_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], -[[C]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e64 [[TMP:v[0-9]+]], [[A]], -[[B]]
-; SI-DENORM-SLOWFMAF: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
-
-; SI: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_fsub_2_f32:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mad_f32 v2, v2, -v3, -v4
+; SI-STD-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_fsub_2_f32:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, -v2, v3, -v4
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_fsub_2_f32:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e64 v2, v2, -v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v2, v2, v4
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -365,26 +838,85 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %o
 }
 
 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32_2uses_neg:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-
-; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], -[[B]], -[[C]]
-; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], -[[B]], -[[D]]
-
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], -[[D]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e64 [[TMP:v[0-9]+]], [[A]], -[[B]]
-; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
-; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e32 [[RESULT1:v[0-9]+]],  [[TMP]], [[D]]
-
-; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_neg(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_fsub_2_f32_2uses_neg:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mad_f32 v4, v2, -v3, -v4
+; SI-STD-NEXT:    v_mad_f32 v2, v2, -v3, -v5
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_fsub_2_f32_2uses_neg:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v4, -v2, v3, -v4
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, -v2, v3, -v5
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_fsub_2_f32_2uses_neg:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e64 v2, v2, -v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v2, v4
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v2, v2, v5
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -409,26 +941,85 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_neg(ptr addrspace(1)
 }
 
 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32_2uses_mul:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-
-; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
-; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
-
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
-; SI-DENORM-FASTFMAF-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
-
-; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT0:v[0-9]+]], -[[TMP]], [[C]]
-; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
-
-; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_mul(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-STD-LABEL: combine_to_mad_fsub_2_f32_2uses_mul:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-STD-NEXT:    s_mov_b32 s7, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s6, 0
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-STD-NEXT:    v_mad_f32 v4, -v2, v3, -v4
+; SI-STD-NEXT:    v_mad_f32 v2, v2, v3, -v5
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: combine_to_mad_fsub_2_f32_2uses_mul:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v4, -v2, v3, -v4
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v2, v2, v3, -v5
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: combine_to_mad_fsub_2_f32_2uses_mul:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s7, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e64 v3, -v2, v4
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v2, v2, v5
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -453,27 +1044,149 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_mul(ptr addrspace(1)
 }
 
 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
-
-; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_0_f32:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-
-; SI-STD-SAFE: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-STD-SAFE: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
-; SI-STD-SAFE: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[C]]
-
-; SI-STD-UNSAFE: v_mad_f32 [[RESULT:v[0-9]+]], [[D]], [[E]], -[[C]]
-; SI-STD-UNSAFE: v_mac_f32_e32 [[RESULT]], [[A]], [[B]]
-
-; SI-DENORM: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-DENORM: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
-; SI-DENORM: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[C]]
-
-; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
+; SI-STD-LABEL: aggressive_combine_to_mad_fsub_0_f32:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-STD-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-STD-NEXT:    s_mov_b32 s2, 0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_mov_b32 s3, 0xf000
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v5, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v6, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-STD-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-STD-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-STD-NEXT:    s_cbranch_vccnz .LBB12_2
+; SI-STD-NEXT:  ; %bb.1: ; %normal
+; SI-STD-NEXT:    v_mul_f32_e32 v4, v6, v1
+; SI-STD-NEXT:    v_fma_f32 v4, v2, v3, v4
+; SI-STD-NEXT:    v_sub_f32_e32 v4, v4, v5
+; SI-STD-NEXT:    s_mov_b64 s[2:3], 0
+; SI-STD-NEXT:    s_branch .LBB12_3
+; SI-STD-NEXT:  .LBB12_2:
+; SI-STD-NEXT:    s_mov_b64 s[2:3], -1
+; SI-STD-NEXT:    ; implicit-def: $vgpr4
+; SI-STD-NEXT:  .LBB12_3: ; %Flow
+; SI-STD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-STD-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 vcc, vcc
+; SI-STD-NEXT:    s_cbranch_vccnz .LBB12_5
+; SI-STD-NEXT:  ; %bb.4: ; %aggressive
+; SI-STD-NEXT:    v_mad_f32 v4, v6, v1, -v5
+; SI-STD-NEXT:    v_mac_f32_e32 v4, v2, v3
+; SI-STD-NEXT:  .LBB12_5: ; %exit
+; SI-STD-NEXT:    s_mov_b32 s3, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s2, 0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: aggressive_combine_to_mad_fsub_0_f32:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-DENORM-FASTFMAF-NEXT:    s_cbranch_vccnz .LBB12_2
+; SI-DENORM-FASTFMAF-NEXT:  ; %bb.1: ; %normal
+; SI-DENORM-FASTFMAF-NEXT:    v_mul_f32_e32 v6, v5, v1
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v6, v2, v3, v6
+; SI-DENORM-FASTFMAF-NEXT:    v_sub_f32_e32 v6, v6, v4
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], 0
+; SI-DENORM-FASTFMAF-NEXT:    s_branch .LBB12_3
+; SI-DENORM-FASTFMAF-NEXT:  .LBB12_2:
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], -1
+; SI-DENORM-FASTFMAF-NEXT:    ; implicit-def: $vgpr6
+; SI-DENORM-FASTFMAF-NEXT:  .LBB12_3: ; %Flow
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 vcc, vcc
+; SI-DENORM-FASTFMAF-NEXT:    s_cbranch_vccnz .LBB12_5
+; SI-DENORM-FASTFMAF-NEXT:  ; %bb.4: ; %aggressive
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v1, v5, v1, -v4
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v6, v2, v3, v1
+; SI-DENORM-FASTFMAF-NEXT:  .LBB12_5: ; %exit
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v6, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: aggressive_combine_to_mad_fsub_0_f32:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v1, v5, v1
+; SI-DENORM-SLOWFMAF-NEXT:    v_fma_f32 v1, v3, v4, v1
+; SI-DENORM-SLOWFMAF-NEXT:    s_cbranch_vccnz .LBB12_2
+; SI-DENORM-SLOWFMAF-NEXT:  ; %bb.1: ; %normal
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v1, v2
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_branch .LBB12_3
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB12_2:
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], -1
+; SI-DENORM-SLOWFMAF-NEXT:    ; implicit-def: $vgpr3
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB12_3: ; %Flow
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 vcc, vcc
+; SI-DENORM-SLOWFMAF-NEXT:    s_cbranch_vccnz .LBB12_5
+; SI-DENORM-SLOWFMAF-NEXT:  ; %bb.4: ; %aggressive
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v1, v2
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB12_5: ; %exit
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -510,25 +1223,32 @@ exit:
 
 ; fold (fsub x, (fma y, z, (fmul u, v)))
 ;   -> (fma (fneg y), z, (fma (fneg u), v, x))
-
-; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_1_f32:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-
-; SI-STD: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-STD: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
-; SI-STD: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[A]], [[TMP1]]
-
-; SI-DENORM: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-DENORM: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
-; SI-DENORM: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[A]], [[TMP1]]
-
-; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @aggressive_combine_to_mad_fsub_1_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: aggressive_combine_to_mad_fsub_1_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v6, v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_mul_f32_e32 v5, v5, v6
+; SI-NEXT:    v_fma_f32 v3, v3, v4, v5
+; SI-NEXT:    v_sub_f32_e32 v2, v2, v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -552,33 +1272,150 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_1_f32(ptr addrspace(1)
 }
 
 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
-
-; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_2_f32:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-
-; SI-STD-SAFE: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-STD-SAFE: v_mac_f32_e32 [[TMP0]], [[A]], [[B]]
-; SI-STD-SAFE: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP0]], [[C]]
-
-; SI-STD-UNSAFE: v_mad_f32 [[RESULT:v[0-9]+]], [[D]], [[E]], -[[C]]
-; SI-STD-UNSAFE: v_mac_f32_e32 [[RESULT]], [[A]], [[B]]
-
-; SI-DENORM-FASTFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-DENORM-FASTFMAF: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
-; SI-DENORM-FASTFMAF: v_sub_f32_e32 [[RESULT:v[0-9]+]],  [[TMP1]], [[C]]
-
-; SI-DENORM-SLOWFMAF-DAG: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-DENORM-SLOWFMAF-DAG: v_mul_f32_e32 [[TMP1:v[0-9]+]], [[A]], [[B]]
-; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP1]], [[TMP0]]
-; SI-DENORM-SLOWFMAF: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP2]], [[C]]
-
-; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
+; SI-STD-LABEL: aggressive_combine_to_mad_fsub_2_f32:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-STD-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-STD-NEXT:    s_mov_b32 s2, 0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_mov_b32 s3, 0xf000
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v6, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-STD-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-STD-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-STD-NEXT:    s_cbranch_vccnz .LBB14_2
+; SI-STD-NEXT:  ; %bb.1: ; %normal
+; SI-STD-NEXT:    v_mul_f32_e32 v5, v6, v1
+; SI-STD-NEXT:    v_mac_f32_e32 v5, v2, v3
+; SI-STD-NEXT:    v_sub_f32_e32 v5, v5, v4
+; SI-STD-NEXT:    s_mov_b64 s[2:3], 0
+; SI-STD-NEXT:    s_branch .LBB14_3
+; SI-STD-NEXT:  .LBB14_2:
+; SI-STD-NEXT:    s_mov_b64 s[2:3], -1
+; SI-STD-NEXT:    ; implicit-def: $vgpr5
+; SI-STD-NEXT:  .LBB14_3: ; %Flow
+; SI-STD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-STD-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 vcc, vcc
+; SI-STD-NEXT:    s_cbranch_vccnz .LBB14_5
+; SI-STD-NEXT:  ; %bb.4: ; %aggressive
+; SI-STD-NEXT:    v_mad_f32 v5, v6, v1, -v4
+; SI-STD-NEXT:    v_mac_f32_e32 v5, v2, v3
+; SI-STD-NEXT:  .LBB14_5: ; %exit
+; SI-STD-NEXT:    s_mov_b32 s3, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s2, 0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    buffer_store_dword v5, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: aggressive_combine_to_mad_fsub_2_f32:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-DENORM-FASTFMAF-NEXT:    s_cbranch_vccnz .LBB14_2
+; SI-DENORM-FASTFMAF-NEXT:  ; %bb.1: ; %normal
+; SI-DENORM-FASTFMAF-NEXT:    v_mul_f32_e32 v6, v5, v1
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v6, v2, v3, v6
+; SI-DENORM-FASTFMAF-NEXT:    v_sub_f32_e32 v6, v6, v4
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], 0
+; SI-DENORM-FASTFMAF-NEXT:    s_branch .LBB14_3
+; SI-DENORM-FASTFMAF-NEXT:  .LBB14_2:
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], -1
+; SI-DENORM-FASTFMAF-NEXT:    ; implicit-def: $vgpr6
+; SI-DENORM-FASTFMAF-NEXT:  .LBB14_3: ; %Flow
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 vcc, vcc
+; SI-DENORM-FASTFMAF-NEXT:    s_cbranch_vccnz .LBB14_5
+; SI-DENORM-FASTFMAF-NEXT:  ; %bb.4: ; %aggressive
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v1, v5, v1, -v4
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v6, v2, v3, v1
+; SI-DENORM-FASTFMAF-NEXT:  .LBB14_5: ; %exit
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v6, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: aggressive_combine_to_mad_fsub_2_f32:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v3, v3, v4
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v1, v5, v1
+; SI-DENORM-SLOWFMAF-NEXT:    v_add_f32_e32 v1, v3, v1
+; SI-DENORM-SLOWFMAF-NEXT:    s_cbranch_vccnz .LBB14_2
+; SI-DENORM-SLOWFMAF-NEXT:  ; %bb.1: ; %normal
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v1, v2
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_branch .LBB14_3
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB14_2:
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], -1
+; SI-DENORM-SLOWFMAF-NEXT:    ; implicit-def: $vgpr3
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB14_3: ; %Flow
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 vcc, vcc
+; SI-DENORM-SLOWFMAF-NEXT:    s_cbranch_vccnz .LBB14_5
+; SI-DENORM-SLOWFMAF-NEXT:  ; %bb.4: ; %aggressive
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v1, v2
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB14_5: ; %exit
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -615,33 +1452,150 @@ exit:
 
 ; fold (fsub x, (fmuladd y, z, (fmul u, v)))
 ;   -> (fmuladd (fneg y), z, (fmuladd (fneg u), v, x))
-
-; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_3_f32:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
-; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-
-; SI-STD-SAFE: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-STD-SAFE: v_mac_f32_e32 [[TMP0]], [[B]], [[C]]
-; SI-STD-SAFE: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[A]], [[TMP0]]
-
-; SI-STD-UNSAFE: v_mad_f32 [[TMP:v[0-9]+]], -[[D]], [[E]], [[A]]
-; SI-STD-UNSAFE: v_mad_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP]]
-
-; SI-DENORM-FASTFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-DENORM-FASTFMAF: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
-; SI-DENORM-FASTFMAF: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[A]], [[TMP1]]
-
-; SI-DENORM-SLOWFMAF-DAG: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
-; SI-DENORM-SLOWFMAF-DAG: v_mul_f32_e32 [[TMP1:v[0-9]+]], [[B]], [[C]]
-; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP1]], [[TMP0]]
-; SI-DENORM-SLOWFMAF: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[A]], [[TMP2]]
-
-; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
+; SI-STD-LABEL: aggressive_combine_to_mad_fsub_3_f32:
+; SI-STD:       ; %bb.0:
+; SI-STD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-STD-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-STD-NEXT:    s_mov_b32 s2, 0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    s_mov_b32 s3, 0xf000
+; SI-STD-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v5, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-STD-NEXT:    s_waitcnt vmcnt(0)
+; SI-STD-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-STD-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-STD-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-STD-NEXT:    s_cbranch_vccnz .LBB15_2
+; SI-STD-NEXT:  ; %bb.1: ; %normal
+; SI-STD-NEXT:    v_mul_f32_e32 v6, v5, v1
+; SI-STD-NEXT:    v_mac_f32_e32 v6, v3, v4
+; SI-STD-NEXT:    v_sub_f32_e32 v6, v2, v6
+; SI-STD-NEXT:    s_mov_b64 s[2:3], 0
+; SI-STD-NEXT:    s_branch .LBB15_3
+; SI-STD-NEXT:  .LBB15_2:
+; SI-STD-NEXT:    s_mov_b64 s[2:3], -1
+; SI-STD-NEXT:    ; implicit-def: $vgpr6
+; SI-STD-NEXT:  .LBB15_3: ; %Flow
+; SI-STD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-STD-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-STD-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-STD-NEXT:    s_mov_b64 vcc, vcc
+; SI-STD-NEXT:    s_cbranch_vccnz .LBB15_5
+; SI-STD-NEXT:  ; %bb.4: ; %aggressive
+; SI-STD-NEXT:    v_mad_f32 v1, -v5, v1, v2
+; SI-STD-NEXT:    v_mad_f32 v6, -v3, v4, v1
+; SI-STD-NEXT:  .LBB15_5: ; %exit
+; SI-STD-NEXT:    s_mov_b32 s3, 0xf000
+; SI-STD-NEXT:    s_mov_b32 s2, 0
+; SI-STD-NEXT:    v_mov_b32_e32 v1, 0
+; SI-STD-NEXT:    buffer_store_dword v6, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT:    s_endpgm
+;
+; SI-DENORM-FASTFMAF-LABEL: aggressive_combine_to_mad_fsub_3_f32:
+; SI-DENORM-FASTFMAF:       ; %bb.0:
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-DENORM-FASTFMAF-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-DENORM-FASTFMAF-NEXT:    s_cbranch_vccnz .LBB15_2
+; SI-DENORM-FASTFMAF-NEXT:  ; %bb.1: ; %normal
+; SI-DENORM-FASTFMAF-NEXT:    v_mul_f32_e32 v6, v5, v1
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v6, v3, v4, v6
+; SI-DENORM-FASTFMAF-NEXT:    v_sub_f32_e32 v6, v2, v6
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], 0
+; SI-DENORM-FASTFMAF-NEXT:    s_branch .LBB15_3
+; SI-DENORM-FASTFMAF-NEXT:  .LBB15_2:
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 s[2:3], -1
+; SI-DENORM-FASTFMAF-NEXT:    ; implicit-def: $vgpr6
+; SI-DENORM-FASTFMAF-NEXT:  .LBB15_3: ; %Flow
+; SI-DENORM-FASTFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-DENORM-FASTFMAF-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-DENORM-FASTFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b64 vcc, vcc
+; SI-DENORM-FASTFMAF-NEXT:    s_cbranch_vccnz .LBB15_5
+; SI-DENORM-FASTFMAF-NEXT:  ; %bb.4: ; %aggressive
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v1, -v5, v1, v2
+; SI-DENORM-FASTFMAF-NEXT:    v_fma_f32 v6, -v3, v4, v1
+; SI-DENORM-FASTFMAF-NEXT:  .LBB15_5: ; %exit
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-FASTFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-FASTFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMAF-NEXT:    buffer_store_dword v6, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT:    s_endpgm
+;
+; SI-DENORM-SLOWFMAF-LABEL: aggressive_combine_to_mad_fsub_3_f32:
+; SI-DENORM-SLOWFMAF:       ; %bb.0:
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dword s6, s[4:5], 0xd
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v5, v[0:1], s[0:3], 0 addr64 offset:12 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:16 glc
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt vmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_bitcmp1_b32 s6, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v3, v3, v4
+; SI-DENORM-SLOWFMAF-NEXT:    v_mul_f32_e32 v1, v5, v1
+; SI-DENORM-SLOWFMAF-NEXT:    v_add_f32_e32 v1, v3, v1
+; SI-DENORM-SLOWFMAF-NEXT:    s_cbranch_vccnz .LBB15_2
+; SI-DENORM-SLOWFMAF-NEXT:  ; %bb.1: ; %normal
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v2, v1
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], 0
+; SI-DENORM-SLOWFMAF-NEXT:    s_branch .LBB15_3
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB15_2:
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 s[2:3], -1
+; SI-DENORM-SLOWFMAF-NEXT:    ; implicit-def: $vgpr3
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB15_3: ; %Flow
+; SI-DENORM-SLOWFMAF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-DENORM-SLOWFMAF-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; SI-DENORM-SLOWFMAF-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b64 vcc, vcc
+; SI-DENORM-SLOWFMAF-NEXT:    s_cbranch_vccnz .LBB15_5
+; SI-DENORM-SLOWFMAF-NEXT:  ; %bb.4: ; %aggressive
+; SI-DENORM-SLOWFMAF-NEXT:    v_sub_f32_e32 v3, v2, v1
+; SI-DENORM-SLOWFMAF-NEXT:  .LBB15_5: ; %exit
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s3, 0xf000
+; SI-DENORM-SLOWFMAF-NEXT:    s_mov_b32 s2, 0
+; SI-DENORM-SLOWFMAF-NEXT:    v_mov_b32_e32 v1, 0
+; SI-DENORM-SLOWFMAF-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1


        
