[llvm] 351d869 - [X86][CostModel] Free a clean narrow zext used as a GEP index (#216256)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 16 21:45:22 PDT 2026
Author: mbhade-amd
Date: 2026-08-17T10:15:18+05:30
New Revision: 351d8690af57ad553284106d812b86e0ac896e90
URL: https://github.com/llvm/llvm-project/commit/351d8690af57ad553284106d812b86e0ac896e90
DIFF: https://github.com/llvm/llvm-project/commit/351d8690af57ad553284106d812b86e0ac896e90.diff
LOG: [X86][CostModel] Free a clean narrow zext used as a GEP index (#216256)
`getCastInstrCost` already treats a narrow (i8/i16) zext of a load as
free (the extension folds into the load). This extends the same
movzx-free reasoning to any narrow zext whose single use is a **GEP
index** with a provably-clean source — a load, a `zeroext` argument, or
known-zero high bits. On x86-64 `[base + index*scale + disp]` reads the
index at full width, so a clean narrow index needs no `movzx`. A dirty
source (e.g. an `i16 add` used only as an index) still needs one and is
unchanged. The cost is consumed by SimplifyCFG's two-entry-PHI fold
budget (`TCK_SizeAndLatency`); overcharging a clean index by one kept
small select diamonds branched on x86.
Test: `llvm/test/Analysis/CostModel/X86/zext-gep-index.ll` — fails on
trunk
(clean known-bits index reports 1), passes with the patch.
---------
Co-authored-by: Cursor <cursoragent at cursor.com>
Added:
llvm/test/Analysis/CostModel/X86/zext-gep-index.ll
Modified:
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 828d79ab0cb45..86f7ee89a68da 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -2429,6 +2429,37 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
+ // A narrow (i8/i16) zero-extension used as a GEP *index* can be folded into
+ // the addressing mode of the consuming memory op, but only if the source is
+ // already materialised zero-extended in a full register. X86's SIB form
+ // [base + index*scale + disp] reads the index at full width and does NOT
+ // zero-extend a narrow index (unlike AArch64's uxtw-extended addressing), so
+ // a "dirty" narrow source (e.g. an i16 add result used only as an index)
+ // still needs a dedicated movzx and is not free. Price it as free only with
+ // positive evidence that no movzx is required.
+ if (ISD == ISD::ZERO_EXTEND && I && I->hasOneUse() && Src->isIntegerTy() &&
+ Src->getScalarSizeInBits() < 32) {
+ const Use &U = *I->use_begin();
+ if (isa<GetElementPtrInst>(U.getUser()) &&
+ U.getOperandNo() != GetElementPtrInst::getPointerOperandIndex()) {
+ const Value *Op = I->getOperand(0);
+ // Clean sources: an extending load, a zeroext argument, or a value whose
+ // high bits are provably zero (e.g. from a shift/mask). These mirror the
+ // proof-based reasoning the middle end uses elsewhere (ValueTracking and
+ // InstCombine's canEvaluateZExtd); we intentionally do NOT treat a merely
+ // multiply-used operand as clean, since that is a guess rather than
+ // proof.
+ if (isa<LoadInst>(Op))
+ return TTI::TCC_Free;
+ if (const auto *A = dyn_cast<Argument>(Op))
+ if (A->hasAttribute(Attribute::ZExt))
+ return TTI::TCC_Free;
+ if (computeKnownBits(Op, I->getDataLayout(), /*AC=*/nullptr, I)
+ .countMinLeadingZeros() > 0)
+ return TTI::TCC_Free;
+ }
+ }
+
// The cost tables include both specific, custom (non-legal) src/dst type
// conversions and generic, legalized types. We test for customs first, before
// falling back to legalization.
diff --git a/llvm/test/Analysis/CostModel/X86/zext-gep-index.ll b/llvm/test/Analysis/CostModel/X86/zext-gep-index.ll
new file mode 100644
index 0000000000000..eb245ad0f16db
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/zext-gep-index.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+; A narrow (i8/i16) zero-extend whose single use is a GEP *index* operand folds
+; into the addressing mode of the consuming memory op (no movzx), but only when
+; the source is already materialised zero-extended in a full register. X86's SIB
+; form [base + index*scale + disp] reads the index at full width and does NOT
+; zero-extend a narrow index (unlike AArch64's uxtw-extended addressing), so a
+; "dirty" narrow source still needs a dedicated movzx and is not free. The cost
+; model prices the zext at 0 only with positive proof that no movzx is required.
+;
+; This cost feeds SimplifyCFG's two-entry-PHI fold budget, which queries
+; TCK_SizeAndLatency (the SizeLat column below).
+
+ at tab = external global [0 x i8]
+
+;; --- Clean sources: no movzx is emitted, so the zext is free (cost 0). ---
+
+; Extending load: the load already leaves a zero-extended value in a full
+; register, so the index folds into the table load with no extra instruction.
+define i8 @clean_i16_load(ptr %q) {
+; CHECK-LABEL: 'clean_i16_load'
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %x = load i16, ptr %q, align 2
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %z = zext i16 %x to i64
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %p = getelementptr inbounds i8, ptr @tab, i64 %z
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %p, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
+;
+ %x = load i16, ptr %q, align 2
+ %z = zext i16 %x to i64
+ %p = getelementptr inbounds i8, ptr @tab, i64 %z
+ %v = load i8, ptr %p, align 1
+ ret i8 %v
+}
+
+; Same for an i8 extending load.
+define i8 @clean_i8_load(ptr %q) {
+; CHECK-LABEL: 'clean_i8_load'
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %x = load i8, ptr %q, align 1
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %z = zext i8 %x to i64
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %p = getelementptr inbounds i8, ptr @tab, i64 %z
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %p, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
+;
+ %x = load i8, ptr %q, align 1
+ %z = zext i8 %x to i64
+ %p = getelementptr inbounds i8, ptr @tab, i64 %z
+ %v = load i8, ptr %p, align 1
+ ret i8 %v
+}
+
+; High bits provably zero from a shift: no movzx needed.
+define i8 @clean_known_bits(i16 zeroext %x) {
+; CHECK-LABEL: 'clean_known_bits'
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %sh = lshr i16 %x, 7
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %z = zext i16 %sh to i64
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %p = getelementptr inbounds i8, ptr @tab, i64 %z
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %p, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
+;
+ %sh = lshr i16 %x, 7
+ %z = zext i16 %sh to i64
+ %p = getelementptr inbounds i8, ptr @tab, i64 %z
+ %v = load i8, ptr %p, align 1
+ ret i8 %v
+}
+
+;; --- Dirty / non-foldable sources: the discount must NOT apply (cost 1). ---
+
+; Dirty i16 add used only as an index: bits 16+ of the register are non-zero,
+; so a dedicated movzwl is emitted.
+define i8 @dirty_i16_add(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: 'dirty_i16_add'
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %d = add i16 %a, %b
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %z = zext i16 %d to i64
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %p = getelementptr inbounds i8, ptr @tab, i64 %z
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %p, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
+;
+ %d = add i16 %a, %b
+ %z = zext i16 %d to i64
+ %p = getelementptr inbounds i8, ptr @tab, i64 %z
+ %v = load i8, ptr %p, align 1
+ ret i8 %v
+}
+
+; The dirty add is also used by an lshr. We do NOT speculate that a shared use
+; keeps the value clean: the index zext of the dirty add still costs a movzwl,
+; while the lshr arm has provably-zero high bits and stays free.
+define i8 @dirty_add_shared(ptr %t, i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: 'dirty_add_shared'
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %d = add i16 %a, %b
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %z = zext i16 %d to i64
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %p = getelementptr inbounds i8, ptr %t, i64 %z
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %p, align 1
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %sh = lshr i16 %d, 7
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %ze = zext i16 %sh to i64
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %pe = getelementptr inbounds i8, ptr %t, i64 %ze
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %ve = load i8, ptr %pe, align 1
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %sum = add i8 %v, %ve
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %sum
+;
+ %d = add i16 %a, %b
+ %z = zext i16 %d to i64
+ %p = getelementptr inbounds i8, ptr %t, i64 %z
+ %v = load i8, ptr %p, align 1
+ %sh = lshr i16 %d, 7
+ %ze = zext i16 %sh to i64
+ %pe = getelementptr inbounds i8, ptr %t, i64 %ze
+ %ve = load i8, ptr %pe, align 1
+ %sum = add i8 %v, %ve
+ ret i8 %sum
+}
+
+; zext not consumed by a GEP (feeds a return): normal cost.
+define i64 @zext_i16_ret(i16 zeroext %x) {
+; CHECK-LABEL: 'zext_i16_ret'
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %z = zext i16 %x to i64
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %z
+;
+ %z = zext i16 %x to i64
+ ret i64 %z
+}
+
+; zext has more than one use: not a pure foldable index, normal cost.
+define i64 @zext_i16_multiuse(i16 zeroext %x) {
+; CHECK-LABEL: 'zext_i16_multiuse'
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %z = zext i16 %x to i64
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %p = getelementptr inbounds i8, ptr @tab, i64 %z
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %p, align 1
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %e = zext i8 %v to i64
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %s = add i64 %z, %e
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %s
+;
+ %z = zext i16 %x to i64
+ %p = getelementptr inbounds i8, ptr @tab, i64 %z
+ %v = load i8, ptr %p, align 1
+ %e = zext i8 %v to i64
+ %s = add i64 %z, %e
+ ret i64 %s
+}
More information about the llvm-commits
mailing list