[llvm] [AArch64][SVE] Use truncating stores whenever possible (PR #196029)
via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 02:16:56 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-selectiondag
Author: Sushant Gokhale (sushgokh)
<details>
<summary>Changes</summary>
For fixed length SVE and fixed length vectors x/y, fold
```
store(concat_vector(truncate(x), truncate(y)))
--> store(truncate(x))
store(truncate(y))
```
---
Full diff: https://github.com/llvm/llvm-project/pull/196029.diff
2 Files Affected:
- (modified) llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (+72-1)
- (modified) llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll (+12-30)
``````````diff
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 2debe9bacf40f..dd71cef1540c1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -537,7 +537,7 @@ namespace {
SDValue replaceStoreOfInsertLoad(StoreSDNode *ST);
bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(SDNode *N);
-
+ SDValue combineStoreConcatTruncVector(StoreSDNode *N);
SDValue visitSTORE(SDNode *N);
SDValue visitATOMIC_STORE(SDNode *N);
SDValue visitLIFETIME_END(SDNode *N);
@@ -23878,6 +23878,74 @@ static SDValue foldToMaskedStore(StoreSDNode *Store, SelectionDAG &DAG,
Store->getAddressingMode());
}
+// store(concat_vector(truncate, truncate))
+// --> store(truncate)
+// store(truncate)
+SDValue DAGCombiner::combineStoreConcatTruncVector(StoreSDNode *ST) {
+ if (!LegalTypes)
+ return SDValue();
+
+ if (ST->isTruncatingStore() || ST->isIndexed())
+ return SDValue();
+
+ SDValue Chain = ST->getChain();
+ SDValue Value = ST->getValue();
+ SDValue Ptr = ST->getBasePtr();
+
+ unsigned Opc = Value.getOpcode();
+ if (Opc != ISD::CONCAT_VECTORS)
+ return SDValue();
+
+ SDValue T1 = Value.getOperand(0);
+ SDValue T2 = Value.getOperand(1);
+ if (T1.getOpcode() != ISD::TRUNCATE || T2.getOpcode() != ISD::TRUNCATE)
+ return SDValue();
+
+ if (!T1.getValueType().isFixedLengthVector())
+ return SDValue();
+
+ if (!T1->hasOneUse() || !T2.hasOneUse())
+ return SDValue();
+
+ EVT LoVT = T1.getOperand(0).getValueType();
+ EVT HiVT = T2.getOperand(0).getValueType();
+ EVT LoMemVT = T1.getValueType();
+ EVT HiMemVT = T2.getValueType();
+ unsigned LoBytes = LoMemVT.getStoreSize();
+ unsigned HiBytes = HiMemVT.getStoreSize();
+ Align LoAlign = ST->getAlign();
+ Align HiAlign = commonAlignment(LoAlign, LoBytes);
+
+ if (!TLI.canCombineTruncStore(T1.getOperand(0).getValueType(),
+ T1.getValueType(), LoAlign,
+ ST->getAddressSpace(), LegalOperations))
+ return SDValue();
+
+ if (!TLI.canCombineTruncStore(T2.getOperand(0).getValueType(),
+ T2.getValueType(), HiAlign,
+ ST->getAddressSpace(), LegalOperations))
+ return SDValue();
+
+ SDLoc DL(ST);
+ SDValue LoPtr = Ptr;
+ SDValue HiPtr =
+ DAG.getMemBasePlusOffset(LoPtr, TypeSize::getFixed(LoBytes), DL);
+
+ MachinePointerInfo LoPI = ST->getPointerInfo();
+ MachinePointerInfo HiPI = ST->getPointerInfo().getWithOffset(LoBytes);
+
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineMemOperand *LoMMO =
+ MF.getMachineMemOperand(ST->getMemOperand(), 0, LoBytes);
+ MachineMemOperand *HiMMO =
+ MF.getMachineMemOperand(ST->getMemOperand(), LoBytes, HiBytes);
+
+ SDValue LoSt = DAG.getStore(Chain, DL, T1, LoPtr, LoMMO);
+ SDValue HiSt = DAG.getStore(Chain, DL, T2, HiPtr, HiMMO);
+
+ return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoSt, HiSt);
+}
+
SDValue DAGCombiner::visitSTORE(SDNode *N) {
StoreSDNode *ST = cast<StoreSDNode>(N);
SDValue Chain = ST->getChain();
@@ -23945,6 +24013,9 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
Chain = ST->getChain();
}
+ if (SDValue R = combineStoreConcatTruncVector(ST))
+ return R;
+
// FIXME: is there such a thing as a truncating indexed store?
if (ST->isTruncatingStore() && ST->isUnindexed() &&
Value.getValueType().isInteger() &&
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll
index 020d5cb53bf21..419fb567158cd 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll
@@ -92,12 +92,8 @@ define void @store_trunc_v8i64i16(ptr %ap, ptr %dest) #0 {
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
-; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s
-; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s
-; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]
-; VBITS_GE_256-NEXT: str q1, [x1]
+; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: st1h { z1.d }, p0, [x1]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: store_trunc_v8i64i16:
@@ -119,12 +115,8 @@ define void @store_trunc_v8i64i32(ptr %ap, ptr %dest) #0 {
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.s, vl4
-; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s
-; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s
-; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s
-; VBITS_GE_256-NEXT: ptrue p0.s, vl8
-; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1]
+; VBITS_GE_256-NEXT: st1w { z0.d }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.d }, p0, [x1]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: store_trunc_v8i64i32:
@@ -147,12 +139,9 @@ define void @store_trunc_v16i32i8(ptr %ap, ptr %dest) #0 {
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
-; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b
-; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]
-; VBITS_GE_256-NEXT: str q1, [x1]
+; VBITS_GE_256-NEXT: mov w8, #8 // =0x8
+; VBITS_GE_256-NEXT: st1b { z0.s }, p0, [x1, x8]
+; VBITS_GE_256-NEXT: st1b { z1.s }, p0, [x1]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: store_trunc_v16i32i8:
@@ -174,12 +163,8 @@ define void @store_trunc_v16i32i16(ptr %ap, ptr %dest) #0 {
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.h, vl8
-; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h
-; VBITS_GE_256-NEXT: ptrue p0.h, vl16
-; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1]
+; VBITS_GE_256-NEXT: st1h { z0.s }, p0, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: st1h { z1.s }, p0, [x1]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: store_trunc_v16i32i16:
@@ -201,12 +186,9 @@ define void @store_trunc_v32i16i8(ptr %ap, ptr %dest) #0 {
; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.b, vl16
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b
-; VBITS_GE_256-NEXT: splice z1.b, p0, z1.b, z0.b
-; VBITS_GE_256-NEXT: ptrue p0.b, vl32
-; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x1]
+; VBITS_GE_256-NEXT: mov w8, #16 // =0x10
+; VBITS_GE_256-NEXT: st1b { z0.h }, p0, [x1, x8]
+; VBITS_GE_256-NEXT: st1b { z1.h }, p0, [x1]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: store_trunc_v32i16i8:
``````````
</details>
https://github.com/llvm/llvm-project/pull/196029
More information about the llvm-commits
mailing list