[llvm] [SystemZTTI][CostModel] Improve SystemZ cost model for scalar Read-Modify-Write Sequence, Fix #189183 (PR #190350)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 3 08:21:54 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-systemz
Author: anoopkg6
<details>
<summary>Changes</summary>
This patch improves the SystemZ cost model to identify Read-Modify-Write sequences
that can be folded into a single instruction (e.g., ASI, NI, OI).
If a load, a scalar arithmetic operation (ADD, SUB, AND, OR, XOR) with an
immediate, and a store all target the same memory location and have no
external uses, the cost of the arithmetic and store insn should bw 0.
This implementation does not include TTI::TCK_RecipThroughput CostKind, as
it causes regression in non-power-2-subvector-extract.ll.
Fixes #<!-- -->189183. (Refer it for example)
---
Full diff: https://github.com/llvm/llvm-project/pull/190350.diff
2 Files Affected:
- (modified) llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp (+68-2)
- (added) llvm/test/Analysis/CostModel/SystemZ/fold-rmw-cost.ll (+147)
``````````diff
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 4322773f4afd6..b453a6bb25d41 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -539,15 +539,44 @@ static unsigned getNumVectorRegs(Type *Ty) {
return ((WideBits % 128U) ? ((WideBits / 128U) + 1) : (WideBits / 128U));
}
+static bool isArithmeticFoldableRMW(const Instruction *I) {
+ if (!I || !isa<BinaryOperator>(I) || !I->hasOneUse())
+ return false;
+
+ Value *Op0 = I->getOperand(0), *Op1 = I->getOperand(1);
+ if (!isa<ConstantInt>(Op0) && !isa<ConstantInt>(Op1))
+ return false;
+
+ Value *V = nullptr;
+ unsigned Opcode = I->getOpcode();
+ if (Opcode == Instruction::And || Opcode == Instruction::Or ||
+ Opcode == Instruction::Xor || Opcode == Instruction::Add)
+ V = isa<ConstantInt>(Op0) ? Op1 : (isa<ConstantInt>(Op1) ? Op0 : nullptr);
+ else if (Opcode == Instruction::Sub && isa<ConstantInt>(Op1))
+ V = Op0;
+ if (!V)
+ return false;
+
+ auto *LI = dyn_cast_or_null<LoadInst>(V);
+ auto *SI = dyn_cast<StoreInst>(*I->users().begin());
+
+ return LI && SI && !LI->isVolatile() && !SI->isVolatile() &&
+ LI->hasOneUse() && LI->getPointerOperand() == SI->getPointerOperand();
+}
+
InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
ArrayRef<const Value *> Args, const Instruction *CxtI) const {
// TODO: Handle more cost kinds.
- if (CostKind != TTI::TCK_RecipThroughput)
+ if (CostKind != TTI::TCK_RecipThroughput) {
+ if (CxtI && Ty && !Ty->isVectorTy() && isArithmeticFoldableRMW(CxtI))
+ return TTI::TCC_Free;
+
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
Op2Info, Args, CxtI);
+ }
// TODO: return a good value for BB-VECTORIZER that includes the
// immediate loads, which we do not want to count for the loop
@@ -1298,6 +1327,35 @@ static bool isBswapIntrinsicCall(const Value *V) {
return false;
}
+static bool isStoreFoldableRMW(const Instruction *I) {
+ auto *SI = dyn_cast_or_null<StoreInst>(I);
+ if (!SI || SI->isVolatile())
+ return false;
+
+ auto *BI = dyn_cast<BinaryOperator>(SI->getValueOperand());
+ if (!BI || !BI->hasOneUse())
+ return false;
+
+ Value *Op0 = BI->getOperand(0), *Op1 = BI->getOperand(1);
+ if (!isa<ConstantInt>(Op0) && !isa<ConstantInt>(Op1))
+ return false;
+
+ unsigned Opcode = BI->getOpcode();
+ Value *V = nullptr;
+ if (Opcode == Instruction::And || Opcode == Instruction::Or ||
+ Opcode == Instruction::Xor || Opcode == Instruction::Add)
+ V = isa<ConstantInt>(Op0) ? Op1 : (isa<ConstantInt>(Op1) ? Op0 : nullptr);
+ else if (Opcode == Instruction::Sub && isa<ConstantInt>(Op1))
+ V = Op0;
+
+ if (!V)
+ return false;
+
+ auto *LI = dyn_cast_or_null<LoadInst>(V);
+ return LI && !LI->isVolatile() &&
+ LI->getPointerOperand() == SI->getPointerOperand();
+}
+
InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
Align Alignment,
unsigned AddressSpace,
@@ -1307,8 +1365,16 @@ InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
assert(!Src->isVoidTy() && "Invalid type");
// TODO: Handle other cost kinds.
- if (CostKind != TTI::TCK_RecipThroughput)
+ if (CostKind != TTI::TCK_RecipThroughput) {
+ // TCK_RecipThroughput causes regressions in the SLP Vectorizer test
+ // non-power-2-subvector-extract.ll. It seems profit analysis does not
+ // favor vectorization by making scalar cost cheaper.
+ if (I && Opcode == Instruction::Store && !Src->isVectorTy()) {
+ if (isStoreFoldableRMW(I))
+ return TTI::TCC_Free;
+ }
return 1;
+ }
if (!Src->isVectorTy() && Opcode == Instruction::Load && I != nullptr) {
// Store the load or its truncated or extended value in FoldedValue.
diff --git a/llvm/test/Analysis/CostModel/SystemZ/fold-rmw-cost.ll b/llvm/test/Analysis/CostModel/SystemZ/fold-rmw-cost.ll
new file mode 100644
index 0000000000000..12199505f2fab
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/SystemZ/fold-rmw-cost.ll
@@ -0,0 +1,147 @@
+; RUN: opt -S -mtriple=s390x-unknown-linux -mcpu=z17 -passes='print<cost-model>' \
+; RUN: -cost-kind=code-size -disable-output %s 2>&1 | FileCheck %s
+;
+; Test the SystemZ cost model for scalar Read-Modify-Write (RMW) folding.
+; A load, a scalar arithmetic operation (ADD, SUB, AND, OR, XOR) with
+; an immediate, and a store all target the same memory location and have
+; no external uses, the cost of the arithmetic and store insn should be 0.
+
+define void @test_and(ptr %p) {
+; CHECK-LABEL: 'test_and'
+; CHECK: cost of 0 {{.*}} and i32
+; CHECK: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = and i32 %v, 1
+ store i32 %res, ptr %p
+ ret void
+}
+
+define void @test_or(ptr %p) {
+; CHECK-LABEL: 'test_or'
+; CHECK: cost of 0 {{.*}} or i32
+; CHECK: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = or i32 %v, 1
+ store i32 %res, ptr %p
+ ret void
+}
+
+define void @test_xor(ptr %p) {
+; CHECK-LABEL: 'test_xor'
+; CHECK: cost of 0 {{.*}} xor i32
+; CHECK: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = xor i32 %v, 1
+ store i32 %res, ptr %p
+ ret void
+}
+
+define void @test_add(ptr %p) {
+; CHECK-LABEL: 'test_add'
+; CHECK: cost of 0 {{.*}} add i32
+; CHECK: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = add i32 %v, 1
+ store i32 %res, ptr %p
+ ret void
+}
+
+define void @test_sub(ptr %p) {
+; CHECK-LABEL: 'test_sub'
+; CHECK: cost of 0 {{.*}} sub i32
+; CHECK: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = sub i32 %v, 1
+ store i32 %res, ptr %p
+ ret void
+}
+
+; Subtraction is not commutative.
+define void @test_sub_neg_imm_lhs(ptr %p) {
+; CHECK-LABEL: 'test_sub_neg_imm_lhs'
+; CHECK-NOT: cost of 0 {{.*}} sub i32
+; CHECK-NOT: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = sub i32 1, %v
+ store i32 %res, ptr %p
+ ret void
+}
+
+; Different Addresses.
+define void @test_diff_addr(ptr %p, ptr %q) {
+; CHECK-LABEL: 'test_diff_addr'
+; CHECK-NOT: cost of 0 {{.*}} add i32
+; CHECK-NOT: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = add i32 %v, 1
+ store i32 %res, ptr %q
+ ret void
+}
+
+; Multi-use of Arithmetic Result.
+define i32 @test_multi_use_arith(ptr %p) {
+; CHECK-LABEL: 'test_multi_use_arith'
+; CHECK-NOT: cost of 0 {{.*}} add i32
+; CHECK-NOT: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = add i32 %v, 1
+ store i32 %res, ptr %p
+ ret i32 %res
+}
+
+; Multi-use of Load Result.
+define i32 @test_multi_use_load(ptr %p) {
+; CHECK-LABEL: 'test_multi_use_load'
+; CHECK-NOT: cost of 0 {{.*}} add i32
+; CHECK-NOT: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = add i32 %v, 1
+ store i32 %res, ptr %p
+ %use2 = add i32 %v, %res
+ ret i32 %use2
+}
+
+; Neither Operand is Immediate.
+define void @test_no_immediate(ptr %p, i32 %reg) {
+; CHECK-LABEL: 'test_no_immediate'
+; CHECK-NOT: cost of 0 {{.*}} add i32
+; CHECK-NOT: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = add i32 %v, %reg
+ store i32 %res, ptr %p
+ ret void
+}
+
+; Both Operands are Immediate.
+define void @test_both_immediate(ptr %p) {
+; CHECK-LABEL: 'test_both_immediate'
+; CHECK-NOT: cost of 0 {{.*}} add i32
+; CHECK-NOT: cost of 0 {{.*}} store i32
+ %v = load i32, ptr %p
+ %res = add i32 5, 10
+ store i32 %res, ptr %p
+ ret void
+}
+
+; Volatile Load/Store.
+define void @test_volatile(ptr %p) {
+; CHECK-LABEL: 'test_volatile'
+; CHECK-NOT: cost of 0 {{.*}} add i32
+; CHECK-NOT: cost of 0 {{.*}} store i32
+ %v = load volatile i32, ptr %p
+ %res = add i32 %v, 1
+ store volatile i32 %res, ptr %p
+ ret void
+}
+
+; Vector types cost should not be changed.
+define void @test_vector_no_fold(<4 x i32> %val, ptr %p) {
+; CHECK-LABEL: 'test_vector_no_fold'
+; CHECK-NOT: cost of 0 {{.*}} add <4 x i32>
+; CHECK-NOT: cost of 0 {{.*}} store <4 x i32>
+ %v = load <4 x i32>, ptr %p
+ %res = add <4 x i32> %v, %val
+ store <4 x i32> %res, ptr %p
+ ret void
+}
+
``````````
</details>
https://github.com/llvm/llvm-project/pull/190350
More information about the llvm-commits
mailing list