[llvm] [RISCV] Fix interleaved + strided costs for e64 elements on rv32 (PR #207927)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 7 01:33:03 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
@llvm/pr-subscribers-backend-risc-v
Author: Luke Lau (lukel97)
<details>
<summary>Changes</summary>
This is the same fix as #<!-- -->176105 for interleaved + strided memory op
costs.
We currently compute the cost of interleaved + strided ops as N *
scalar element memory ops, but on rv32 the scalar memory access will
be on an illegal type for 64 bit elements, even with zve64x. Prevent
it from overcosting by just using TCC_Basic.
---
Full diff: https://github.com/llvm/llvm-project/pull/207927.diff
4 Files Affected:
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp (+2-15)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll (+6-10)
- (added) llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses-i64-rv32.ll (+55)
- (added) llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll (+52)
``````````diff
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 888923fd1471b..a50c1664ad5a1 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1139,11 +1139,8 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
// Otherwise, the cost is proportional to the number of elements (VL *
// Factor ops).
- InstructionCost MemOpCost =
- getMemoryOpCost(Opcode, VTy->getElementType(), Alignment, 0,
- CostKind, {TTI::OK_AnyValue, TTI::OP_None});
unsigned NumLoads = getEstimatedVLFor(VTy);
- return NumLoads * MemOpCost;
+ return NumLoads * TTI::TCC_Basic;
}
}
}
@@ -1303,14 +1300,8 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
InstructionCost
RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
-
- unsigned Opcode = MICA.getID() == Intrinsic::experimental_vp_strided_load
- ? Instruction::Load
- : Instruction::Store;
-
Type *DataTy = MICA.getDataType();
Align Alignment = MICA.getAlignment();
- const Instruction *I = MICA.getInst();
if (!isLegalStridedLoadStore(DataTy, Alignment))
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
@@ -1321,13 +1312,9 @@ RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
// Cost is proportional to the number of memory operations implied. For
// scalable vectors, we use an estimate on that number since we don't
// know exactly what VL will be.
- // FIXME: This will overcost for i64 on rv32 with +zve64x.
auto &VTy = *cast<VectorType>(DataTy);
- InstructionCost MemOpCost =
- getMemoryOpCost(Opcode, VTy.getElementType(), Alignment, 0, CostKind,
- {TTI::OK_AnyValue, TTI::OP_None}, I);
unsigned NumLoads = getEstimatedVLFor(&VTy);
- return NumLoads * MemOpCost;
+ return NumLoads * TTI::TCC_Basic;
}
InstructionCost
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
index 46055d50890f7..aeb9fb179d4ba 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
@@ -232,6 +232,7 @@ define i64 @strided_search(ptr align 8 dereferenceable(14784) %p) {
; RV32-LABEL: define i64 @strided_search(
; RV32-SAME: ptr align 8 dereferenceable(14784) [[P:%.*]]) #[[ATTR0]] {
; RV32-NEXT: [[ENTRY:.*]]:
+; RV32-NEXT: [[TMP2:%.*]] = getelementptr nuw i8, ptr [[P]], i32 88
; RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; RV32-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; RV32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 132, [[TMP1]]
@@ -241,24 +242,19 @@ define i64 @strided_search(ptr align 8 dereferenceable(14784) %p) {
; RV32-NEXT: [[N_MOD_VF:%.*]] = urem i64 132, [[TMP3]]
; RV32-NEXT: [[N_VEC:%.*]] = sub i64 132, [[N_MOD_VF]]
; RV32-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 112
-; RV32-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; RV32-NEXT: [[TMP6:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP5]], splat (i64 112)
-; RV32-NEXT: [[TMP7:%.*]] = mul nuw nsw i64 112, [[TMP3]]
-; RV32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP7]], i64 0
-; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV32-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; RV32-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
-; RV32-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], <vscale x 2 x i64> [[VEC_IND]]
-; RV32-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, <vscale x 2 x ptr> [[TMP8]], i64 88
-; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP9]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> poison)
+; RV32-NEXT: [[TMP6:%.*]] = trunc i64 [[INDEX]] to i32
+; RV32-NEXT: [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 112
+; RV32-NEXT: [[TMP8:%.*]] = getelementptr nuw i8, ptr [[TMP2]], i32 [[TMP7]]
+; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i32(ptr align 8 [[TMP8]], i32 112, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; RV32-NEXT: [[TMP10:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
; RV32-NEXT: [[TMP11:%.*]] = freeze <vscale x 2 x i1> [[TMP10]]
; RV32-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP11]])
; RV32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; RV32-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV32-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; RV32-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV32: [[VECTOR_BODY_INTERIM]]:
; RV32-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses-i64-rv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses-i64-rv32.ll
new file mode 100644
index 0000000000000..095a3353d8204
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses-i64-rv32.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=riscv32 -mattr=+v,-optimized-nf2-segment-load-store < %s -S | FileCheck %s
+
+%i64.2 = type {i64, i64}
+define void @i64_factor_2(ptr noalias %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @i64_factor_2(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[SMAX]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[I64_2:%.*]], ptr [[SRC]], i64 [[INDEX]], i32 0
+; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT: [[WIDE_VP_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VP_LOAD]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[I64_2]], ptr [[DST]], i64 [[INDEX]], i32 0
+; CHECK-NEXT: [[INTERLEAVE_EVL1:%.*]] = mul nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr align 8 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP2]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %p0 = getelementptr inbounds %i64.2, ptr %src, i64 %i, i32 0
+ %p1 = getelementptr inbounds %i64.2, ptr %src, i64 %i, i32 1
+ %l0 = load i64, ptr %p0
+ %l1 = load i64, ptr %p1
+ %q0 = getelementptr inbounds %i64.2, ptr %dst, i64 %i, i32 0
+ %q1 = getelementptr inbounds %i64.2, ptr %dst, i64 %i, i32 1
+ store i64 %l0, ptr %q0
+ store i64 %l1, ptr %q1
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll
new file mode 100644
index 0000000000000..6955e409355e2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=riscv32 -mattr=+v < %s -S | FileCheck %s
+
+define void @single_constant_stride_int_scaled(ptr %p) {
+; CHECK-LABEL: define void @single_constant_stride_int_scaled(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw <vscale x 2 x i32> [[VEC_IND]], splat (i32 3)
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i32> [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shl i32 [[INDEX]], 6
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP5]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i32(ptr align 8 [[TMP6]], i32 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x ptr> align 8 [[WIDE_GEP]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP1]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+
+ %offset = mul nsw nuw i32 %iv, 8
+ %gep = getelementptr i64, ptr %p, i32 %offset
+ %x = load i64, ptr %gep
+ %y = add i64 %x, 1
+ store i64 %y, ptr %gep
+
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/207927
More information about the llvm-commits
mailing list