[llvm] 54702ff - [X86] Fold `B + (-C)*A` into `neg + lea` (#215145)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 05:17:21 PDT 2026
Author: Nikita Taranov
Date: 2026-08-19T12:17:17Z
New Revision: 54702ff00a04bdb731aeff42039562b50042863c
URL: https://github.com/llvm/llvm-project/commit/54702ff00a04bdb731aeff42039562b50042863c
DIFF: https://github.com/llvm/llvm-project/commit/54702ff00a04bdb731aeff42039562b50042863c.diff
LOG: [X86] Fold `B + (-C)*A` into `neg + lea` (#215145)
Closes: #37287
Added:
llvm/test/CodeGen/X86/neg-shl-lea-32.ll
llvm/test/CodeGen/X86/neg-shl-lea.ll
Modified:
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll
llvm/test/CodeGen/X86/ipra-local-linkage-2.ll
llvm/test/CodeGen/X86/large-gep-scale.ll
llvm/test/CodeGen/X86/lea-opt.ll
llvm/test/CodeGen/X86/urem-vector-lkk.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index d078117061677..c3fc43c2bc6db 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -31,6 +31,7 @@
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include <cstdint>
+#include <optional>
using namespace llvm;
@@ -2036,8 +2037,11 @@ bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) {
}
// Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has
- // a smaller encoding and avoids a scaled-index.
- if (AM.Scale == 2 &&
+ // a smaller encoding and avoids a scaled-index. Not valid when the index is
+ // negated: this copies the index into the base, but only the index is negated
+ // when the address is emitted, so the result would be index + (-index) - that
+ // is, zero - rather than (-index) * 2.
+ if (AM.Scale == 2 && !AM.NegateIndex &&
AM.BaseType == X86ISelAddressMode::RegBase &&
AM.Base_Reg.getNode() == nullptr) {
AM.Base_Reg = AM.IndexReg;
@@ -2789,12 +2793,14 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
break;
case ISD::SUB: {
- // Given A-B, if A can be completely folded into the address and
- // the index field with the index field unused, use -B as the index.
- // This is a win if a has multiple parts that can be folded into
- // the address. Also, this saves a mov if the base register has
- // other uses, since it avoids a two-address sub instruction, however
- // it costs an additional mov if the index register has other uses.
+ // Given A-B, if A can be completely folded into the address leaving the
+ // index field unused, use -B as the index. This is a win if A has multiple
+ // parts that can be folded into the address. Also, this saves a mov if the
+ // base register has other uses, since it avoids a two-address sub
+ // instruction, however it costs an additional mov if the index register
+ // has other uses.
+ // B may itself be a constant shift, in which case the shift folds into
+ // the scale - see below.
// Add an artificial use to this node so that we can keep track of
// it if it gets CSE'd with a
diff erent node.
@@ -2816,21 +2822,56 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
int Cost = 0;
SDValue RHS = N.getOperand(1);
+
+ // A-(B<<C) can use -B as a scaled index for C in [1,3], which folds the
+ // shift into the address as well as the subtract. When B is not a foldable
+ // shift, NegScale stays empty and this is the plain A-B fold, which only
+ // breaks even on instruction count - a-b is mov+sub either way. Absorbing
+ // the shift saves one:
+ //
+ // a - (b << 2) movq %rdi, %rax -> negq %rsi
+ // shlq $2, %rsi leaq (%rdi,%rsi,4), %rax
+ // subq %rsi, %rax
+ //
+ // That pays for the negate, so drop the cost by one.
+ std::optional<unsigned> NegScale;
+ if (RHS.getOpcode() == ISD::SHL && RHS.hasOneUse()) {
+ if (auto *ShAmt = dyn_cast<ConstantSDNode>(RHS.getOperand(1))) {
+ uint64_t ShVal = ShAmt->getZExtValue();
+ if (ShVal >= 1 && ShVal <= 3) {
+ NegScale = 1u << ShVal;
+ RHS = RHS.getOperand(0);
+ --Cost;
+ }
+ }
+ }
+
// If the RHS involves a register with multiple uses, this
// transformation incurs an extra mov, due to the neg instruction
- // clobbering its operand.
+ // clobbering its operand. The CopyFromReg part of that is a guess -
+ // SelectionDAG is per-block, so uses elsewhere are invisible - and it is
+ // not applied to a folded shift, where it is wrong often enough to matter.
+ // The multiple-use part still is; see @y_outlives_lea.
if (!RHS.getNode()->hasOneUse() ||
- RHS.getNode()->getOpcode() == ISD::CopyFromReg ||
+ (!NegScale && RHS.getNode()->getOpcode() == ISD::CopyFromReg) ||
RHS.getNode()->getOpcode() == ISD::TRUNCATE ||
RHS.getNode()->getOpcode() == ISD::ANY_EXTEND ||
(RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND &&
RHS.getOperand(0).getValueType() == MVT::i32))
++Cost;
- // If the base is a register with multiple uses, this
- // transformation may save a mov.
- if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
- !AM.Base_Reg.getNode()->hasOneUse()) ||
- AM.BaseType == X86ISelAddressMode::FrameIndexBase)
+ // A - (A << C), where the base is itself the value being negated.
+ bool BaseIsNegatedValue = NegScale &&
+ AM.BaseType == X86ISelAddressMode::RegBase &&
+ AM.Base_Reg == RHS;
+ // If the base is a register with multiple uses, this transformation may
+ // save a mov - but not for BaseIsNegatedValue, where the baseline emits the
+ // shift non-destructively into another register and the SUB writes A in
+ // place, so there is no copy for the LEA to save. The copy the NEG needs
+ // there is charged by the multiple-use test above.
+ if (((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
+ !AM.Base_Reg.getNode()->hasOneUse()) ||
+ AM.BaseType == X86ISelAddressMode::FrameIndexBase) &&
+ !BaseIsNegatedValue)
--Cost;
// If the folded LHS was interesting, this transformation saves
// address arithmetic.
@@ -2849,7 +2890,7 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
// was an unprofitable LEA.
AM.IndexReg = RHS;
AM.NegateIndex = true;
- AM.Scale = 1;
+ AM.Scale = NegScale.value_or(1);
return false;
}
diff --git a/llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll b/llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll
index 41fa34667af86..4c0a85e3968ec 100644
--- a/llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll
+++ b/llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll
@@ -22,11 +22,9 @@ entry:
define void @neg_8bit_2(i8 %int8) {
; NDD-LABEL: neg_8bit_2:
; NDD: # %bb.0: # %entry
-; NDD-NEXT: # kill: def $edi killed $edi def $rdi
; NDD-NEXT: addb %dil, %dil, %al # encoding: [0x62,0xf4,0x7c,0x18,0x00,0xff]
-; NDD-NEXT: negb %al, %al # encoding: [0x62,0xf4,0x7c,0x18,0xf6,0xd8]
-; NDD-NEXT: leal 1(%rdi,%rax), %eax # encoding: [0x8d,0x44,0x07,0x01]
-; NDD-NEXT: # kill: def $al killed $al killed $eax
+; NDD-NEXT: subb %al, %dil, %al # encoding: [0x62,0xf4,0x7c,0x18,0x28,0xc7]
+; NDD-NEXT: incb %al # EVEX TO LEGACY Compression encoding: [0xfe,0xc0]
; NDD-NEXT: mulb %dil # encoding: [0x40,0xf6,0xe7]
; NDD-NEXT: testb %al, %al # encoding: [0x84,0xc0]
; NDD-NEXT: retq # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/ipra-local-linkage-2.ll b/llvm/test/CodeGen/X86/ipra-local-linkage-2.ll
index 05d3f70820fb0..8c41c3caeea42 100644
--- a/llvm/test/CodeGen/X86/ipra-local-linkage-2.ll
+++ b/llvm/test/CodeGen/X86/ipra-local-linkage-2.ll
@@ -127,10 +127,8 @@ define void @caller_use_esi(i32 %X) nounwind ssp {
; X86-NEXT: movl 8(%ebp), %eax
; X86-NEXT: movl __stack_chk_guard, %ecx
; X86-NEXT: movl %ecx, 16(%esi)
-; X86-NEXT: movl %esp, %ecx
-; X86-NEXT: shll $2, %eax
-; X86-NEXT: subl %eax, %ecx
-; X86-NEXT: movl %ecx, %esp
+; X86-NEXT: negl %eax
+; X86-NEXT: leal (%esp,%eax,4), %esp
; X86-NEXT: movl %esi, %eax
; X86-NEXT: pushl %eax
; X86-NEXT: calll callee_clobber_esi
diff --git a/llvm/test/CodeGen/X86/large-gep-scale.ll b/llvm/test/CodeGen/X86/large-gep-scale.ll
index 7b672c9a36767..277f1716a08d1 100644
--- a/llvm/test/CodeGen/X86/large-gep-scale.ll
+++ b/llvm/test/CodeGen/X86/large-gep-scale.ll
@@ -4,7 +4,10 @@
; After scaling, this type doesn't fit in memory. Codegen should generate
; correct addressing still.
-; CHECK: shll $2, %edx
+; The scale is 2147483647*4, which is -4 mod 2^32, so this is u - 4*t and
+; folds into a negated scaled index.
+; CHECK: negl %edx
+; CHECK-NEXT: leal (%ecx,%edx,4), %eax
define fastcc ptr @_ada_smkr(ptr %u, i32 %t) nounwind {
%x = getelementptr [2147483647 x i32], ptr %u, i32 %t, i32 0
diff --git a/llvm/test/CodeGen/X86/lea-opt.ll b/llvm/test/CodeGen/X86/lea-opt.ll
index 88712328e54a7..58747d99048ff 100644
--- a/llvm/test/CodeGen/X86/lea-opt.ll
+++ b/llvm/test/CodeGen/X86/lea-opt.ll
@@ -311,9 +311,10 @@ sw.epilog: ; preds = %sw.bb.2, %sw.bb.1,
define i32 @test5(i32 %x, i32 %y) #0 {
; CHECK-LABEL: test5:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: addl %esi, %esi
-; CHECK-NEXT: subl %esi, %eax
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,2), %eax
; CHECK-NEXT: retq
entry:
%mul = mul nsw i32 %y, -2
@@ -338,9 +339,10 @@ entry:
define i32 @test7(i32 %x, i32 %y) #0 {
; CHECK-LABEL: test7:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: shll $2, %esi
-; CHECK-NEXT: subl %esi, %eax
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,4), %eax
; CHECK-NEXT: retq
entry:
%mul = mul nsw i32 %y, -4
@@ -365,9 +367,10 @@ entry:
define i32 @test9(i32 %x, i32 %y) #0 {
; CHECK-LABEL: test9:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: addl %esi, %esi
-; CHECK-NEXT: subl %esi, %eax
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,2), %eax
; CHECK-NEXT: retq
entry:
%mul = mul nsw i32 -2, %y
@@ -392,9 +395,10 @@ entry:
define i32 @test11(i32 %x, i32 %y) #0 {
; CHECK-LABEL: test11:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: shll $2, %esi
-; CHECK-NEXT: subl %esi, %eax
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,4), %eax
; CHECK-NEXT: retq
entry:
%mul = mul nsw i32 -4, %y
@@ -418,9 +422,8 @@ entry:
define i64 @test13(i64 %x, i64 %y) #0 {
; CHECK-LABEL: test13:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shlq $2, %rsi
-; CHECK-NEXT: subq %rsi, %rax
+; CHECK-NEXT: negq %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi,4), %rax
; CHECK-NEXT: retq
entry:
%mul = mul nsw i64 -4, %y
@@ -444,9 +447,10 @@ entry:
define zeroext i16 @test15(i16 zeroext %x, i16 zeroext %y) #0 {
; CHECK-LABEL: test15:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: shll $3, %esi
-; CHECK-NEXT: subl %esi, %eax
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,8), %eax
; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/neg-shl-lea-32.ll b/llvm/test/CodeGen/X86/neg-shl-lea-32.ll
new file mode 100644
index 0000000000000..15886ef40f27b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/neg-shl-lea-32.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu | FileCheck %s
+
+; 32-bit coverage for the sub(X, shl(Y, C)) -> neg + scaled-index LEA fold in
+; matchAddressRecursively. See neg-shl-lea.ll for the 64-bit tests. There is no
+; 64-bit-only gate: the cost model is the same, and where it fires here the
+; result is no longer than the SHL + SUB it replaces.
+
+define i32 @shl2_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: shl2_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: negl %ecx
+; CHECK-NEXT: leal (%eax,%ecx,4), %eax
+; CHECK-NEXT: retl
+ %s = shl i32 %y, 2
+ %r = sub i32 %x, %s
+ ret i32 %r
+}
+
+define i32 @mul_form(i32 %x, i32 %y) {
+; CHECK-LABEL: mul_form:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: negl %ecx
+; CHECK-NEXT: leal (%eax,%ecx,4), %eax
+; CHECK-NEXT: retl
+ %m = mul i32 %y, -4
+ %r = add i32 %x, %m
+ ret i32 %r
+}
+
+; The scale-2 no-base case, which is where the index negation interacts with
+; the lea(,%reg,2) -> lea(%reg,%reg) rewrite.
+define i32 @scale2_no_base(ptr %p) {
+; CHECK-LABEL: scale2_no_base:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl (%eax), %eax
+; CHECK-NEXT: negl %eax
+; CHECK-NEXT: leal 64(,%eax,2), %eax
+; CHECK-NEXT: retl
+ %y = load i32, ptr %p
+ %s = shl i32 %y, 1
+ %r = sub i32 64, %s
+ ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/neg-shl-lea.ll b/llvm/test/CodeGen/X86/neg-shl-lea.ll
new file mode 100644
index 0000000000000..4ff34e121518b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/neg-shl-lea.ll
@@ -0,0 +1,266 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+; sub(X, shl(Y, C)) for C in [1,3] becomes neg + a scaled-index LEA, folding
+; both the shift and the subtract into the address. See issue #37287.
+;
+; This happens in matchAddressRecursively's ISD::SUB case, which already used
+; -B as the index for A-B; the shift now folds into the scale as well.
+
+define i64 @shl1(i64 %x, i64 %y) {
+; CHECK-LABEL: shl1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: negq %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi,2), %rax
+; CHECK-NEXT: retq
+ %s = shl i64 %y, 1
+ %r = sub i64 %x, %s
+ ret i64 %r
+}
+
+define i64 @shl2(i64 %x, i64 %y) {
+; CHECK-LABEL: shl2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: negq %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi,4), %rax
+; CHECK-NEXT: retq
+ %s = shl i64 %y, 2
+ %r = sub i64 %x, %s
+ ret i64 %r
+}
+
+define i64 @shl3(i64 %x, i64 %y) {
+; CHECK-LABEL: shl3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: negq %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi,8), %rax
+; CHECK-NEXT: retq
+ %s = shl i64 %y, 3
+ %r = sub i64 %x, %s
+ ret i64 %r
+}
+
+define i32 @shl2_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: shl2_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,4), %eax
+; CHECK-NEXT: retq
+ %s = shl i32 %y, 2
+ %r = sub i32 %x, %s
+ ret i32 %r
+}
+
+; C >= 4 is outside the LEA scale range. (C = 0 cannot reach this code - a
+; shift by zero is folded away first - so there is nothing to test there.)
+define i32 @shl4(i32 %x, i32 %y) {
+; CHECK-LABEL: shl4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: shll $4, %esi
+; CHECK-NEXT: subl %esi, %eax
+; CHECK-NEXT: retq
+ %s = shl i32 %y, 4
+ %r = sub i32 %x, %s
+ ret i32 %r
+}
+
+; The multiply spelling from the original report, at each usable scale. The mul
+; reaches the DAG intact here, so these cover the mul -> shl -> address path.
+define i32 @mul_form(i32 %x, i32 %y) {
+; CHECK-LABEL: mul_form:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,4), %eax
+; CHECK-NEXT: retq
+ %m = mul i32 %y, -4
+ %r = add i32 %x, %m
+ ret i32 %r
+}
+
+define i32 @mul_form_2(i32 %x, i32 %y) {
+; CHECK-LABEL: mul_form_2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,2), %eax
+; CHECK-NEXT: retq
+ %m = mul i32 %y, -2
+ %r = add i32 %x, %m
+ ret i32 %r
+}
+
+define i32 @mul_form_8(i32 %x, i32 %y) {
+; CHECK-LABEL: mul_form_8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,8), %eax
+; CHECK-NEXT: retq
+ %m = mul i32 %y, -8
+ %r = add i32 %x, %m
+ ret i32 %r
+}
+
+; -3 is not a power of two, so the mul never becomes a shl and this never
+; applies. LLVM's mov+lea+sub already matches GCC's lea+sub+lea at three
+; instructions.
+define i32 @mul_form_3(i32 %x, i32 %y) {
+; CHECK-LABEL: mul_form_3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: leal (%rsi,%rsi,2), %ecx
+; CHECK-NEXT: subl %ecx, %eax
+; CHECK-NEXT: retq
+ %m = mul i32 %y, -3
+ %r = add i32 %x, %m
+ ret i32 %r
+}
+
+; The already-negated spelling, add(x, shl(0 - y, n)). DAGCombiner canonicalises
+; this to the sub form before ISel, so it arrives here the same way.
+define i32 @add_neg_form(i32 %x, i32 %y) {
+; CHECK-LABEL: add_neg_form:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rdi,%rsi,4), %eax
+; CHECK-NEXT: retq
+ %n = sub i32 0, %y
+ %s = shl i32 %n, 2
+ %r = add i32 %x, %s
+ ret i32 %r
+}
+
+; X is still live after the sub, so the two-address SUB needs a MOV to preserve
+; it, while the LEA writes a fresh register. The fold saves one instruction.
+define i64 @x_live_after(i64 %x, i64 %y) {
+; CHECK-LABEL: x_live_after:
+; CHECK: # %bb.0:
+; CHECK-NEXT: negq %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi,8), %rax
+; CHECK-NEXT: xorq %rdi, %rax
+; CHECK-NEXT: retq
+ %s = shl i64 %y, 3
+ %t = sub i64 %x, %s
+ %r = xor i64 %t, %x
+ ret i64 %r
+}
+
+; Y has another use, so the NEG would clobber a value that is still needed. Here
+; that use is a store, which is scheduled before the NEG, so folding would in
+; fact have been a win by one instruction. hasOneUse() counts uses without
+; ordering them, so it cannot tell this apart from @y_outlives_lea, where
+; folding genuinely costs one.
+define i32 @y_multi_use(i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: y_multi_use:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: movl %esi, (%rdx)
+; CHECK-NEXT: shll $2, %esi
+; CHECK-NEXT: subl %esi, %eax
+; CHECK-NEXT: retq
+ store i32 %y, ptr %p
+ %s = shl i32 %y, 2
+ %r = sub i32 %x, %s
+ ret i32 %r
+}
+
+; Y outlives the LEA: the add consumes the sub's result, so Y cannot be used up
+; before the NEG and would need a copy. X dies at the sub, so the baseline pays
+; no copy of its own - folding here would cost an instruction, and the
+; multiple-use test is what declines it.
+define void @y_outlives_lea(i64 %x, i64 %y, ptr %p) {
+; CHECK-LABEL: y_outlives_lea:
+; CHECK: # %bb.0:
+; CHECK-NEXT: leaq (,%rsi,4), %rax
+; CHECK-NEXT: subq %rax, %rdi
+; CHECK-NEXT: addq %rsi, %rdi
+; CHECK-NEXT: movq %rdi, (%rdx)
+; CHECK-NEXT: retq
+ %s = shl i64 %y, 2
+ %r = sub i64 %x, %s
+ %z = add i64 %r, %y
+ store i64 %z, ptr %p
+ ret void
+}
+
+; X - (X << C): X is also the base, so NEG cannot write it in place and the fold
+; needs a copy, which exactly cancels the absorbed shift - three instructions
+; either way (the folded form is four bytes shorter). The model declines because
+; BaseIsNegatedValue suppresses the "base has multiple uses" discount, which
+; would otherwise double-count that copy. Suppressing it changes nothing in this
+; function, but avoids regressions of 9 and 4 instructions in
+; vector-idiv-udiv-128 and -256.
+define i32 @x_minus_x_shl(i32 %x) {
+; CHECK-LABEL: x_minus_x_shl:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: leal (,%rax,8), %ecx
+; CHECK-NEXT: subl %ecx, %eax
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT: retq
+ %s = shl i32 %x, 3
+ %r = sub i32 %x, %s
+ ret i32 %r
+}
+
+; X is a load. It is materialized into a register and used as the LEA base;
+; the shift still folds into the scale, so this is the same length either way.
+define i32 @x_is_load(ptr %p, i32 %y) {
+; CHECK-LABEL: x_is_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: negl %esi
+; CHECK-NEXT: leal (%rax,%rsi,4), %eax
+; CHECK-NEXT: retq
+ %x = load i32, ptr %p
+ %s = shl i32 %y, 2
+ %r = sub i32 %x, %s
+ ret i32 %r
+}
+
+; Scale 2 with the LHS folded entirely into the displacement, so the address
+; has no base. matchAddress rewrites lea(,%reg,2) into lea(%reg,%reg) for the
+; shorter encoding, which is only valid when the index is not negated - doing
+; it here would compute base + (-index), i.e. 64, instead of 64 - 2*y.
+define i32 @scale2_no_base(ptr %p) {
+; CHECK-LABEL: scale2_no_base:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: negl %eax
+; CHECK-NEXT: leal 64(,%rax,2), %eax
+; CHECK-NEXT: retq
+ %y = load i32, ptr %p
+ %s = shl i32 %y, 1
+ %r = sub i32 64, %s
+ ret i32 %r
+}
+
+ at g = dso_local global [64 x i8] zeroinitializer
+
+; No shift to fold. The RHS is a single-use CopyFromReg, and that still counts
+; against the fold even though the symbolic displacement would otherwise pay
+; for it: NEG clobbers a live-in argument. Folding would give neg + a baseless
+; leaq - the same two instructions, but 11 bytes against 8, since that LEA needs
+; a SIB and a disp32. This mainly pins the boundary; the shape is common enough
+; in real code that acting on the fold is a measurable loss.
+define i64 @no_shift_copyfromreg(i64 %n) {
+; CHECK-LABEL: no_shift_copyfromreg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl $g+56, %eax
+; CHECK-NEXT: subq %rdi, %rax
+; CHECK-NEXT: retq
+ %q = ptrtoint ptr getelementptr (i8, ptr @g, i64 56) to i64
+ %r = sub i64 %q, %n
+ ret i64 %r
+}
diff --git a/llvm/test/CodeGen/X86/urem-vector-lkk.ll b/llvm/test/CodeGen/X86/urem-vector-lkk.ll
index 7b3d0c9e12cfc..20fe4c0c12075 100644
--- a/llvm/test/CodeGen/X86/urem-vector-lkk.ll
+++ b/llvm/test/CodeGen/X86/urem-vector-lkk.ll
@@ -263,8 +263,8 @@ define <4 x i64> @fold_urem_i64(<4 x i64> %x) {
; SSE2-NEXT: addq %rdx, %rax
; SSE2-NEXT: shrq $4, %rax
; SSE2-NEXT: leaq (%rax,%rax,2), %rdx
-; SSE2-NEXT: shlq $3, %rdx
-; SSE2-NEXT: subq %rdx, %rax
+; SSE2-NEXT: negq %rdx
+; SSE2-NEXT: leaq (%rax,%rdx,8), %rax
; SSE2-NEXT: addq %rcx, %rax
; SSE2-NEXT: movq %rax, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
@@ -302,8 +302,8 @@ define <4 x i64> @fold_urem_i64(<4 x i64> %x) {
; SSE4-NEXT: addq %rdx, %rax
; SSE4-NEXT: shrq $4, %rax
; SSE4-NEXT: leaq (%rax,%rax,2), %rdx
-; SSE4-NEXT: shlq $3, %rdx
-; SSE4-NEXT: subq %rdx, %rax
+; SSE4-NEXT: negq %rdx
+; SSE4-NEXT: leaq (%rax,%rdx,8), %rax
; SSE4-NEXT: addq %rcx, %rax
; SSE4-NEXT: movq %rax, %xmm2
; SSE4-NEXT: pextrq $1, %xmm1, %rcx
@@ -341,8 +341,8 @@ define <4 x i64> @fold_urem_i64(<4 x i64> %x) {
; AVX1-NEXT: addq %rdx, %rax
; AVX1-NEXT: shrq $4, %rax
; AVX1-NEXT: leaq (%rax,%rax,2), %rdx
-; AVX1-NEXT: shlq $3, %rdx
-; AVX1-NEXT: subq %rdx, %rax
+; AVX1-NEXT: negq %rdx
+; AVX1-NEXT: leaq (%rax,%rdx,8), %rax
; AVX1-NEXT: addq %rcx, %rax
; AVX1-NEXT: vmovq %rax, %xmm2
; AVX1-NEXT: vpextrq $1, %xmm1, %rcx
More information about the llvm-commits
mailing list