[llvm] 7d1cb18 - [AMDGPU] Fix FP/BP spills when MUBUF constant offset exceeded
Austin Kerbow via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 14:12:27 PDT 2020
Author: Austin Kerbow
Date: 2020-08-13T14:12:00-07:00
New Revision: 7d1cb187fbee336e831309ea22d4f2e273331d6d
URL: https://github.com/llvm/llvm-project/commit/7d1cb187fbee336e831309ea22d4f2e273331d6d
DIFF: https://github.com/llvm/llvm-project/commit/7d1cb187fbee336e831309ea22d4f2e273331d6d.diff
LOG: [AMDGPU] Fix FP/BP spills when MUBUF constant offset exceeded
If we need a scratch register for the spill don't use the same scratch
register that is being used for the MBUF offset.
Reviewed By: arsenm
Differential Revision: https://reviews.llvm.org/D85772
Added:
Modified:
llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
llvm/test/CodeGen/AMDGPU/stack-realign.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index fdccd833b36b..d73b29053ede 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -158,6 +158,10 @@ static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
return;
}
+ // Don't clobber the TmpVGPR if we also need a scratch reg for the stack
+ // offset in the spill.
+ LiveRegs.addReg(SpillReg);
+
MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister(
MF->getRegInfo(), LiveRegs, AMDGPU::VGPR_32RegClass);
@@ -176,6 +180,8 @@ static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
.addImm(0) // dlc
.addImm(0) // swz
.addMemOperand(MMO);
+
+ LiveRegs.removeReg(SpillReg);
}
static void buildEpilogReload(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
diff --git a/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll b/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
index 440555dc0692..3016d99fc8c1 100644
--- a/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
+++ b/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
@@ -472,7 +472,7 @@ define void @ipra_call_with_stack() #0 {
; GCN: v_readfirstlane_b32 s33, [[TMP_VGPR2]]
; GCN: s_mov_b64 exec, [[COPY_EXEC2]]
; GCN: s_setpc_b64
-define void @callee_need_to_spill_fp_to_memory() #1 {
+define void @callee_need_to_spill_fp_to_memory() #3 {
call void asm sideeffect "; clobber nonpreserved SGPRs",
"~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
@@ -483,29 +483,7 @@ define void @callee_need_to_spill_fp_to_memory() #1 {
"~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
- ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}
- ,~{v40},~{v41},~{v42},~{v43},~{v44},~{v45},~{v46},~{v47},~{v48},~{v49}
- ,~{v50},~{v51},~{v52},~{v53},~{v54},~{v55},~{v56},~{v57},~{v58},~{v59}
- ,~{v60},~{v61},~{v62},~{v63},~{v64},~{v65},~{v66},~{v67},~{v68},~{v69}
- ,~{v70},~{v71},~{v72},~{v73},~{v74},~{v75},~{v76},~{v77},~{v78},~{v79}
- ,~{v80},~{v81},~{v82},~{v83},~{v84},~{v85},~{v86},~{v87},~{v88},~{v89}
- ,~{v90},~{v91},~{v92},~{v93},~{v94},~{v95},~{v96},~{v97},~{v98},~{v99}
- ,~{v100},~{v101},~{v102},~{v103},~{v104},~{v105},~{v106},~{v107},~{v108},~{v109}
- ,~{v110},~{v111},~{v112},~{v113},~{v114},~{v115},~{v116},~{v117},~{v118},~{v119}
- ,~{v120},~{v121},~{v122},~{v123},~{v124},~{v125},~{v126},~{v127},~{v128},~{v129}
- ,~{v130},~{v131},~{v132},~{v133},~{v134},~{v135},~{v136},~{v137},~{v138},~{v139}
- ,~{v140},~{v141},~{v142},~{v143},~{v144},~{v145},~{v146},~{v147},~{v148},~{v149}
- ,~{v150},~{v151},~{v152},~{v153},~{v154},~{v155},~{v156},~{v157},~{v158},~{v159}
- ,~{v160},~{v161},~{v162},~{v163},~{v164},~{v165},~{v166},~{v167},~{v168},~{v169}
- ,~{v170},~{v171},~{v172},~{v173},~{v174},~{v175},~{v176},~{v177},~{v178},~{v179}
- ,~{v180},~{v181},~{v182},~{v183},~{v184},~{v185},~{v186},~{v187},~{v188},~{v189}
- ,~{v190},~{v191},~{v192},~{v193},~{v194},~{v195},~{v196},~{v197},~{v198},~{v199}
- ,~{v200},~{v201},~{v202},~{v203},~{v204},~{v205},~{v206},~{v207},~{v208},~{v209}
- ,~{v210},~{v211},~{v212},~{v213},~{v214},~{v215},~{v216},~{v217},~{v218},~{v219}
- ,~{v220},~{v221},~{v222},~{v223},~{v224},~{v225},~{v226},~{v227},~{v228},~{v229}
- ,~{v230},~{v231},~{v232},~{v233},~{v234},~{v235},~{v236},~{v237},~{v238},~{v239}
- ,~{v240},~{v241},~{v242},~{v243},~{v244},~{v245},~{v246},~{v247},~{v248},~{v249}
- ,~{v250},~{v251},~{v252},~{v253},~{v254},~{v255}"()
+ ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"()
ret void
}
@@ -525,7 +503,7 @@ define void @callee_need_to_spill_fp_to_memory() #1 {
; GCN: v_readfirstlane_b32 s33, [[TMP_VGPR2]]
; GCN: s_mov_b64 exec, [[COPY_EXEC2]]
; GCN: s_setpc_b64
-define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #1 {
+define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 {
call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
"~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
@@ -542,32 +520,43 @@ define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #1 {
"~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
- ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}
- ,~{v41},~{v42},~{v43},~{v44},~{v45},~{v46},~{v47},~{v48},~{v49}
- ,~{v50},~{v51},~{v52},~{v53},~{v54},~{v55},~{v56},~{v57},~{v58},~{v59}
- ,~{v60},~{v61},~{v62},~{v63},~{v64},~{v65},~{v66},~{v67},~{v68},~{v69}
- ,~{v70},~{v71},~{v72},~{v73},~{v74},~{v75},~{v76},~{v77},~{v78},~{v79}
- ,~{v80},~{v81},~{v82},~{v83},~{v84},~{v85},~{v86},~{v87},~{v88},~{v89}
- ,~{v90},~{v91},~{v92},~{v93},~{v94},~{v95},~{v96},~{v97},~{v98},~{v99}
- ,~{v100},~{v101},~{v102},~{v103},~{v104},~{v105},~{v106},~{v107},~{v108},~{v109}
- ,~{v110},~{v111},~{v112},~{v113},~{v114},~{v115},~{v116},~{v117},~{v118},~{v119}
- ,~{v120},~{v121},~{v122},~{v123},~{v124},~{v125},~{v126},~{v127},~{v128},~{v129}
- ,~{v130},~{v131},~{v132},~{v133},~{v134},~{v135},~{v136},~{v137},~{v138},~{v139}
- ,~{v140},~{v141},~{v142},~{v143},~{v144},~{v145},~{v146},~{v147},~{v148},~{v149}
- ,~{v150},~{v151},~{v152},~{v153},~{v154},~{v155},~{v156},~{v157},~{v158},~{v159}
- ,~{v160},~{v161},~{v162},~{v163},~{v164},~{v165},~{v166},~{v167},~{v168},~{v169}
- ,~{v170},~{v171},~{v172},~{v173},~{v174},~{v175},~{v176},~{v177},~{v178},~{v179}
- ,~{v180},~{v181},~{v182},~{v183},~{v184},~{v185},~{v186},~{v187},~{v188},~{v189}
- ,~{v190},~{v191},~{v192},~{v193},~{v194},~{v195},~{v196},~{v197},~{v198},~{v199}
- ,~{v200},~{v201},~{v202},~{v203},~{v204},~{v205},~{v206},~{v207},~{v208},~{v209}
- ,~{v210},~{v211},~{v212},~{v213},~{v214},~{v215},~{v216},~{v217},~{v218},~{v219}
- ,~{v220},~{v221},~{v222},~{v223},~{v224},~{v225},~{v226},~{v227},~{v228},~{v229}
- ,~{v230},~{v231},~{v232},~{v233},~{v234},~{v235},~{v236},~{v237},~{v238},~{v239}
- ,~{v240},~{v241},~{v242},~{v243},~{v244},~{v245},~{v246},~{v247},~{v248},~{v249}
- ,~{v250},~{v251},~{v252},~{v253},~{v254},~{v255}"()
+ ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"()
+ ret void
+}
+
+; If the size of the offset exceeds the MUBUF offset field we need another
+; scratch VGPR to hold the offset.
+; GCN-LABEL: {{^}}spill_fp_to_memory_scratch_reg_needed_mubuf_offset
+; GCN: s_or_saveexec_b64 s[4:5], -1
+; GCN: v_mov_b32_e32 v0, s33
+; GCN-NOT: v_mov_b32_e32 v0, 0x100c
+; GCN-NEXT: v_mov_b32_e32 v1, 0x100c
+; GCN-NEXT: buffer_store_dword v0, v1, s[0:3], s32 offen
+define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset([4096 x i8] addrspace(5)* byval align 4 %arg) #3 {
+ %alloca = alloca i32, addrspace(5)
+ store volatile i32 0, i32 addrspace(5)* %alloca
+
+ call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
+ "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
+ ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
+ ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
+ ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
+ ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
+ ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
+ ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
+ ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
+ ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
+ ,~{s100},~{s101},~{s102},~{s39},~{vcc}"()
+
+ call void asm sideeffect "; clobber all VGPRs except CSR v40",
+ "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
+ ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
+ ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
+ ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"()
ret void
}
attributes #0 = { nounwind }
attributes #1 = { nounwind "frame-pointer"="all" }
attributes #2 = { nounwind "frame-pointer"="non-leaf" }
+attributes #3 = { nounwind "frame-pointer"="all" "amdgpu-waves-per-eu"="6,6" }
diff --git a/llvm/test/CodeGen/AMDGPU/stack-realign.ll b/llvm/test/CodeGen/AMDGPU/stack-realign.ll
index 193f99731331..74b53802ef5b 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-realign.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-realign.ll
@@ -254,8 +254,78 @@ define void @no_free_scratch_sgpr_for_bp_copy(<32 x i32> %a, i32 %b) #0 {
ret void
}
+define void @no_free_regs_spill_bp_to_memory(<32 x i32> %a, i32 %b) #5 {
+; If there are no free SGPRs or VGPRs available we must spill the BP to memory.
+
+; GCN-LABEL: no_free_regs_spill_bp_to_mem
+; GCN: s_or_saveexec_b64 s[4:5], -1
+; GCN: v_mov_b32_e32 v0, s33
+; GCN: buffer_store_dword v0, off, s[0:3], s32
+; GCN: v_mov_b32_e32 v0, s34
+; GCN-DAG: buffer_store_dword v0, off, s[0:3], s32
+ %local_val = alloca i32, align 128, addrspace(5)
+ store volatile i32 %b, i32 addrspace(5)* %local_val, align 128
+
+ call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
+ "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
+ ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
+ ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
+ ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
+ ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
+ ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
+ ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
+ ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
+ ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
+ ,~{s100},~{s101},~{s102},~{s39},~{vcc}"() #0
+
+ call void asm sideeffect "; clobber all VGPRs",
+ "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
+ ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
+ ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
+ ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}" () #0
+ ret void
+}
+
+define void @spill_bp_to_memory_scratch_reg_needed_mubuf_offset(<32 x i32> %a, i32 %b, [4096 x i8] addrspace(5)* byval align 4 %arg) #5 {
+; If the size of the offset exceeds the MUBUF offset field we need another
+; scratch VGPR to hold the offset.
+
+; GCN-LABEL: spill_bp_to_memory_scratch_reg_needed_mubuf_offset
+; GCN: s_or_saveexec_b64 s[4:5], -1
+; GCN: v_mov_b32_e32 v0, s33
+; GCN-NOT: v_mov_b32_e32 v0, 0x1088
+; GCN-NEXT: v_mov_b32_e32 v1, 0x1088
+; GCN-NEXT: buffer_store_dword v0, v1, s[0:3], s32 offen
+; GCN: v_mov_b32_e32 v0, s34
+; GCN-NOT: v_mov_b32_e32 v0, 0x1090
+; GCN-NEXT: v_mov_b32_e32 v1, 0x1090
+; GCN-NEXT: buffer_store_dword v0, v1, s[0:3], s32 offen
+ %local_val = alloca i32, align 128, addrspace(5)
+ store volatile i32 %b, i32 addrspace(5)* %local_val, align 128
+
+ call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
+ "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
+ ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
+ ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
+ ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
+ ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
+ ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
+ ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
+ ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
+ ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
+ ,~{s100},~{s101},~{s102},~{s39},~{vcc}"() #0
+
+ call void asm sideeffect "; clobber all VGPRs",
+ "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
+ ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
+ ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
+ ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"() #0
+ ret void
+}
+
attributes #0 = { noinline nounwind }
attributes #1 = { noinline nounwind "stackrealign" }
attributes #2 = { noinline nounwind alignstack=4 }
attributes #3 = { noinline nounwind "no-realign-stack" }
attributes #4 = { noinline nounwind "frame-pointer"="all"}
+attributes #5 = { noinline nounwind "amdgpu-waves-per-eu"="6,6" }
More information about the llvm-commits
mailing list