[llvm] 1ad91be - [CodeGen] relax kill copy hoist restriction for vreg to phys reg copies with folded loads (#190304)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 7 19:08:23 PDT 2026
Author: Takashi Idobe
Date: 2026-04-08T10:08:18+08:00
New Revision: 1ad91beba9fe14c152370a5d116aa020478a01c3
URL: https://github.com/llvm/llvm-project/commit/1ad91beba9fe14c152370a5d116aa020478a01c3
DIFF: https://github.com/llvm/llvm-project/commit/1ad91beba9fe14c152370a5d116aa020478a01c3.diff
LOG: [CodeGen] relax kill copy hoist restriction for vreg to phys reg copies with folded loads (#190304)
Resolves: https://github.com/llvm/llvm-project/issues/62452
Currently, `TwoAddressInstructionPass` has a blanket rule against moving
kill copies, since many copies are better handled later by coalescing.
However, that rule is too strict when the kill is a virtual register to
physical register copy and the current two-address instruction has a
folded load. In that case, keeping the copy in place can force the pass
to break the folded rm form into a mov rm + op rr, even though the
physical register copy itself cannot be coalesced away in the usual
sense.
This fixes a missed optimization where a folded IMUL64rm was rewritten
into MOV64rm + IMUL64rr because a later $rax = COPY %src was kept in
place for mul.
Added:
llvm/test/CodeGen/X86/two-address-subreg-to-reg-kill.mir
Modified:
llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
llvm/test/CodeGen/X86/mul-lohi-no-implicit-copy.ll
llvm/test/CodeGen/X86/pr78897.ll
llvm/test/CodeGen/X86/tailcallstack64.ll
llvm/test/CodeGen/X86/tailccstack64.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index ace5c6e49596e..351aa6179d86a 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -1148,10 +1148,27 @@ bool TwoAddressInstructionImpl::rescheduleKillAboveMI(
} else {
KillMI = LV->getVarInfo(Reg).findKill(MBB);
}
- if (!KillMI || MI == KillMI || KillMI->isCopy() || KillMI->isCopyLike())
- // Don't mess with copies, they may be coalesced later.
+ if (!KillMI || MI == KillMI)
return false;
+ if (KillMI->isCopyLike()) {
+ if (!MI->mayLoad())
+ return false;
+
+ Register CopySrcReg, CopyDstReg;
+ bool IsCopySrcPhys, IsCopyDstPhys;
+ // Most copies are better left for coalescing. Allow moving only the
+ // case of a kill-copy from a source virtual register into a
+ // physical register when the current two-address instruction has a folded
+ // load; that preserves the memory form and avoids introducing a load+copy.
+ if (!isCopyToReg(*KillMI, CopySrcReg, CopyDstReg, IsCopySrcPhys,
+ IsCopyDstPhys))
+ return false;
+
+ if (CopySrcReg != Reg || IsCopySrcPhys || !IsCopyDstPhys)
+ return false;
+ }
+
Register DstReg;
if (isTwoAddrUse(*KillMI, Reg, DstReg))
return false;
diff --git a/llvm/test/CodeGen/X86/mul-lohi-no-implicit-copy.ll b/llvm/test/CodeGen/X86/mul-lohi-no-implicit-copy.ll
index 0135ec230cf28..6815a4d178f22 100644
--- a/llvm/test/CodeGen/X86/mul-lohi-no-implicit-copy.ll
+++ b/llvm/test/CodeGen/X86/mul-lohi-no-implicit-copy.ll
@@ -37,3 +37,45 @@ define i64 @mul64_no_implicit_copy(i64 %a0) nounwind {
%a3 = extractvalue { i64, i1 } %a2, 0
ret i64 %a3
}
+
+define i64 @mul64_add_hi_order_a(ptr %x, i64 %y) {
+; CHECK-LABEL: mul64_add_hi_order_a:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: imulq 8(%rdi), %rsi
+; CHECK-NEXT: mulq (%rdi)
+; CHECK-NEXT: leaq (%rdx,%rsi), %rax
+; CHECK-NEXT: retq
+ %p1 = getelementptr inbounds i64, ptr %x, i64 1
+ %qv = load i64, ptr %p1, align 8
+ %q = mul i64 %qv, %y
+ %pv = load i64, ptr %x, align 8
+ %pv.zext = zext i64 %pv to i128
+ %y.zext = zext i64 %y to i128
+ %prod = mul nuw i128 %pv.zext, %y.zext
+ %prod.hi = lshr i128 %prod, 64
+ %p = trunc i128 %prod.hi to i64
+ %sum = add i64 %q, %p
+ ret i64 %sum
+}
+
+define i64 @mul64_add_hi_order_b(ptr %x, i64 %y) {
+; CHECK-LABEL: mul64_add_hi_order_b:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: mulq (%rdi)
+; CHECK-NEXT: imulq 8(%rdi), %rsi
+; CHECK-NEXT: leaq (%rsi,%rdx), %rax
+; CHECK-NEXT: retq
+ %pv = load i64, ptr %x, align 8
+ %pv.zext = zext i64 %pv to i128
+ %y.zext = zext i64 %y to i128
+ %prod = mul nuw i128 %pv.zext, %y.zext
+ %prod.hi = lshr i128 %prod, 64
+ %p = trunc i128 %prod.hi to i64
+ %p1 = getelementptr inbounds i64, ptr %x, i64 1
+ %qv = load i64, ptr %p1, align 8
+ %q = mul i64 %qv, %y
+ %sum = add i64 %q, %p
+ ret i64 %sum
+}
diff --git a/llvm/test/CodeGen/X86/pr78897.ll b/llvm/test/CodeGen/X86/pr78897.ll
index db77baa7ff8a3..f5e8d77a87a3f 100644
--- a/llvm/test/CodeGen/X86/pr78897.ll
+++ b/llvm/test/CodeGen/X86/pr78897.ll
@@ -136,12 +136,12 @@ define <16 x i8> @produceShuffleVectorForByte(i8 zeroext %0) nounwind {
; X64-SSE42: # %bb.0: # %entry
; X64-SSE42-NEXT: movd %edi, %xmm0
; X64-SSE42-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-SSE42-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
+; X64-SSE42-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; X64-SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE42-NEXT: pxor %xmm1, %xmm1
+; X64-SSE42-NEXT: pcmpeqb %xmm0, %xmm1
+; X64-SSE42-NEXT: movdqa %xmm1, %xmm0
; X64-SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; X64-SSE42-NEXT: pxor %xmm0, %xmm0
-; X64-SSE42-NEXT: pcmpeqb %xmm1, %xmm0
-; X64-SSE42-NEXT: movdqa {{.*#+}} xmm1 = [17,17,17,17,17,17,17,17,u,u,u,u,u,u,u,u]
-; X64-SSE42-NEXT: pand %xmm0, %xmm1
; X64-SSE42-NEXT: movq %xmm1, %rax
; X64-SSE42-NEXT: movabsq $1229782938247303440, %rcx # imm = 0x1111111111111110
; X64-SSE42-NEXT: movabsq $76861433640456465, %rdx # imm = 0x111111111111111
diff --git a/llvm/test/CodeGen/X86/tailcallstack64.ll b/llvm/test/CodeGen/X86/tailcallstack64.ll
index 77295ee0ff871..ef57ede471886 100644
--- a/llvm/test/CodeGen/X86/tailcallstack64.ll
+++ b/llvm/test/CodeGen/X86/tailcallstack64.ll
@@ -5,12 +5,12 @@
; CHECK: subq ${{24|72|80}}, %rsp
; Check that lowered arguments on the stack do not overwrite each other.
-; Add %in1 %p1 to a
diff erent temporary register (%eax).
-; CHECK: movl [[A1:32|144]](%rsp), [[R1:%e..|%r.*d]]
-; Move param %in1 to temp register (%r10d).
+; Move %p1 to a temporary register (%eax).
+; CHECK: movl {{%edi|%ecx}}, [[R1:%e..|%r.*d]]
+; Fold the load of %in1 into the add.
+; CHECK: addl [[A1:32|144]](%rsp), [[R1]]
+; Move param %in2 to temp register (%r10d).
; CHECK: movl [[A2:40|152]](%rsp), [[R2:%[a-z0-9]+]]
-; Add %in1 %p1 to a
diff erent temporary register (%eax).
-; CHECK: addl {{%edi|%ecx}}, [[R1]]
; Move param %in2 to stack.
; CHECK-DAG: movl [[R2]], [[A1]](%rsp)
; Move result of addition to stack.
diff --git a/llvm/test/CodeGen/X86/tailccstack64.ll b/llvm/test/CodeGen/X86/tailccstack64.ll
index bcedea5e16384..9b3754a9ee27d 100644
--- a/llvm/test/CodeGen/X86/tailccstack64.ll
+++ b/llvm/test/CodeGen/X86/tailccstack64.ll
@@ -6,12 +6,12 @@
; CHECK: subq ${{24|72|80}}, %rsp
; Check that lowered arguments on the stack do not overwrite each other.
-; Add %in1 %p1 to a
diff erent temporary register (%eax).
-; CHECK: movl [[A1:32|144]](%rsp), [[R1:%e..|%r.*d]]
-; Move param %in1 to temp register (%r10d).
+; Move %p1 to a temporary register (%eax).
+; CHECK: movl {{%edi|%ecx}}, [[R1:%e..|%r.*d]]
+; Fold the load of %in1 into the add.
+; CHECK: addl [[A1:32|144]](%rsp), [[R1]]
+; Move param %in2 to temp register (%r10d).
; CHECK: movl [[A2:40|152]](%rsp), [[R2:%[a-z0-9]+]]
-; Add %in1 %p1 to a
diff erent temporary register (%eax).
-; CHECK: addl {{%edi|%ecx}}, [[R1]]
; Move param %in2 to stack.
; CHECK-DAG: movl [[R2]], [[A1]](%rsp)
; Move result of addition to stack.
diff --git a/llvm/test/CodeGen/X86/two-address-subreg-to-reg-kill.mir b/llvm/test/CodeGen/X86/two-address-subreg-to-reg-kill.mir
new file mode 100644
index 0000000000000..dbbcff0498ded
--- /dev/null
+++ b/llvm/test/CodeGen/X86/two-address-subreg-to-reg-kill.mir
@@ -0,0 +1,108 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu --run-pass=livevars,twoaddressinstruction %s -o - | FileCheck %s
+
+# Positive case: IMUL32rm with %1 as a tied source should be hoisted
+# before the pass, subreg_to_reg kills a 32-bit vreg (%1) and writes to
+# physical $eax, which allows IMUL32rm can use %eax.
+---
+name: subreg_to_reg_kill_hoist
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: gr64 }
+ - { id: 1, class: gr32 }
+ - { id: 2, class: gr32 }
+liveins:
+ - { reg: '$rdi', virtual-reg: '%0' }
+ - { reg: '$esi', virtual-reg: '%1' }
+body: |
+ bb.0:
+ liveins: $rdi, $esi
+
+ ; CHECK-LABEL: name: subreg_to_reg_kill_hoist
+ ; CHECK: [[COPY1:%[0-9]+]]:gr32 = COPY killed $esi
+ ; CHECK-NEXT: dead $rax = SUBREG_TO_REG [[COPY1]], %subreg.sub_32bit
+ ; CHECK: IMUL32rm
+ %0:gr64 = COPY killed $rdi
+ %1:gr32 = COPY killed $esi
+ dead %2:gr32 = IMUL32rm %1, %0, 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load (s32) from `ptr undef`, align 4)
+ dead $rax = SUBREG_TO_REG killed %1, %subreg.sub_32bit
+ RET 0
+
+---
+# Negative case: subreg_to_reg with a vreg is not hoisted.
+# The pass keeps the IMUL32rm as is and MOV32rm + IMUL32rr is emitted instead.
+name: subreg_to_reg_virtual_dst_no_hoist
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: gr64 }
+ - { id: 1, class: gr32 }
+ - { id: 2, class: gr32 }
+ - { id: 3, class: gr64 }
+liveins:
+ - { reg: '$rdi', virtual-reg: '%0' }
+ - { reg: '$esi', virtual-reg: '%1' }
+body: |
+ bb.0:
+ liveins: $rdi, $esi
+
+ ; CHECK-LABEL: name: subreg_to_reg_virtual_dst_no_hoist
+ ; CHECK: MOV32rm
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY
+ ; CHECK-NEXT: dead [[COPY2]]:gr32 = IMUL32rr [[COPY2]], %1, implicit-def dead $eflags
+ %0:gr64 = COPY killed $rdi
+ %1:gr32 = COPY killed $esi
+ dead %2:gr32 = IMUL32rm %1, %0, 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load (s32) from `ptr undef`, align 4)
+ dead %3:gr64 = SUBREG_TO_REG killed %1, %subreg.sub_32bit
+ RET 0
+
+---
+# Positive case: COPY to a physical register is hoisted above IMUL32rm.
+name: copy_kill_hoist
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: gr64 }
+ - { id: 1, class: gr32 }
+ - { id: 2, class: gr32 }
+liveins:
+ - { reg: '$rdi', virtual-reg: '%0' }
+ - { reg: '$esi', virtual-reg: '%1' }
+body: |
+ bb.0:
+ liveins: $rdi, $esi
+
+ ; CHECK-LABEL: name: copy_kill_hoist
+ ; CHECK: [[COPY1:%[0-9]+]]:gr32 = COPY killed $esi
+ ; CHECK-NEXT: dead $eax = COPY [[COPY1]]
+ ; CHECK: IMUL32rm
+ %0:gr64 = COPY killed $rdi
+ %1:gr32 = COPY killed $esi
+ dead %2:gr32 = IMUL32rm %1, %0, 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load (s32) from `ptr undef`, align 4)
+ dead $eax = COPY killed %1
+ RET 0
+
+---
+# Negative case: COPY with a vreg destination is not hoisted.
+name: copy_virtual_dst_no_hoist
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: gr64 }
+ - { id: 1, class: gr32 }
+ - { id: 2, class: gr32 }
+ - { id: 3, class: gr32 }
+liveins:
+ - { reg: '$rdi', virtual-reg: '%0' }
+ - { reg: '$esi', virtual-reg: '%1' }
+body: |
+ bb.0:
+ liveins: $rdi, $esi
+
+ ; CHECK-LABEL: name: copy_virtual_dst_no_hoist
+ ; CHECK: MOV32rm
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY
+ ; CHECK-NEXT: dead [[COPY2]]:gr32 = IMUL32rr [[COPY2]], %1, implicit-def dead $eflags
+ %0:gr64 = COPY killed $rdi
+ %1:gr32 = COPY killed $esi
+ dead %2:gr32 = IMUL32rm %1, %0, 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load (s32) from `ptr undef`, align 4)
+ dead %3:gr32 = COPY killed %1
+ RET 0
+...
More information about the llvm-commits
mailing list