[llvm] [Hexagon] Correctly split masked loads/stores after widening (PR #190689)
Alexey Karyakin via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 6 14:25:22 PDT 2026
https://github.com/quic-akaryaki created https://github.com/llvm/llvm-project/pull/190689
Non-power of 2 masked loads and stores are first widened to a power of 2 size. If the original vector was larger than the single HVX register size, they become multiples of the double HVX register size (e.g. v64f32, v128f32). Since double vector types are legal but do not support masked operations, they are split during custom lowering.
MemoryVT is not changed during widening and the original code assumed that MemoryVT is the same as the operand type and always equal to the double HVX register size.
For example, @llvm.masked.store.v80f32.p0 is first widened to v128f32, then split into two v64f32 (MemoryVT = v64f32 and v16f32). The first half was lowered correctly but the second one could not be split due to the unexpected v16f32 MemoryVT.
>From 303ff2dc3ffeb33f880fdfbc1ff9297c8ec75517 Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki at qti.qualcomm.com>
Date: Mon, 6 Apr 2026 13:06:31 -0700
Subject: [PATCH] [Hexagon] Correctly split masked loads/stores after widening
Non-power of 2 masked loads and stores are first widened to a power of 2
size. If the original vector was larger than the single HVX register
size, they become multiples of the double HVX register size
(e.g. v64f32, v128f32). Since double vector types are legal but do not
support masked operations, they are split during custom lowering.
MemoryVT is not changed during widening and the original code assumed
that MemoryVT is the same as the operand type and always equal to the
double HVX register size.
For example, @llvm.masked.store.v80f32.p0 is first widened to v128f32,
then split into two v64f32 (MemoryVT = v64f32 and v16f32). The first
half was lowered correctly but the second one could not be split due to
the unexpected v16f32 MemoryVT.
Change-Id: I95caa3572cd49057626a3d3263acbc7cb7b37b04
---
.../Target/Hexagon/HexagonISelLoweringHVX.cpp | 95 ++++++++++++-------
1 file changed, 61 insertions(+), 34 deletions(-)
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 385c2520f46d9..afef82b59acd7 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -3443,41 +3443,60 @@ HexagonTargetLowering::SplitVectorOp(SDValue Op, SelectionDAG &DAG) const {
SDValue
HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
auto *MemN = cast<MemSDNode>(Op.getNode());
+ unsigned MemOpc = MemN->getOpcode();
+ EVT MemTy = MemN->getMemoryVT();
- if (!MemN->getMemoryVT().isSimple())
+ if ((MemOpc == ISD::STORE || MemOpc == ISD::LOAD) &&
+ (!MemTy.isSimple() || !isHvxPairTy(MemTy.getSimpleVT())))
return Op;
- MVT MemTy = MemN->getMemoryVT().getSimpleVT();
- if (!isHvxPairTy(MemTy))
- return Op;
+ EVT ValueType;
+ if (MemOpc == ISD::STORE)
+ ValueType = ty(cast<StoreSDNode>(Op)->getValue());
+ else if (MemOpc == ISD::MSTORE)
+ ValueType = ty(cast<MaskedStoreSDNode>(Op)->getValue());
+ else // ISD::LOAD, ISD::MLOAD.
+ ValueType = MemN->getValueType(0);
+
+ EVT LoVT, HiVT;
+ std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(ValueType);
+
+ EVT LoMemVT, HiMemVT;
+ bool HiIsEmpty = false;
+ std::tie(LoMemVT, HiMemVT) =
+ DAG.GetDependentSplitDestVTs(MemTy, LoVT, &HiIsEmpty);
+
+ uint64_t LoSize = LoMemVT.getSizeInBits().getFixedValue() / 8;
+ uint64_t HiSize = HiMemVT.getSizeInBits().getFixedValue() / 8;
const SDLoc &dl(Op);
- unsigned HwLen = Subtarget.getVectorLength();
- MVT SingleTy = typeSplit(MemTy).first;
SDValue Chain = MemN->getChain();
SDValue Base0 = MemN->getBasePtr();
SDValue Base1 =
- DAG.getMemBasePlusOffset(Base0, TypeSize::getFixed(HwLen), dl);
- unsigned MemOpc = MemN->getOpcode();
+ DAG.getMemBasePlusOffset(Base0, TypeSize::getFixed(LoSize), dl);
MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
if (MachineMemOperand *MMO = MemN->getMemOperand()) {
MachineFunction &MF = DAG.getMachineFunction();
- uint64_t MemSize = (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE)
- ? (uint64_t)MemoryLocation::UnknownSize
- : HwLen;
- MOp0 = MF.getMachineMemOperand(MMO, 0, MemSize);
- MOp1 = MF.getMachineMemOperand(MMO, HwLen, MemSize);
+ auto MemSize = [=](uint64_t Size) {
+ return (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE)
+ ? (uint64_t)MemoryLocation::UnknownSize
+ : Size;
+ };
+ MOp0 = MF.getMachineMemOperand(MMO, 0, MemSize(LoSize));
+ MOp1 = MF.getMachineMemOperand(MMO, LoSize, MemSize(HiSize));
}
if (MemOpc == ISD::LOAD) {
assert(cast<LoadSDNode>(Op)->isUnindexed());
- SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
- SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
+ SDValue Load0 = DAG.getLoad(LoVT, dl, Chain, Base0, MOp0);
+ SDValue Load1 = DAG.getLoad(HiVT, dl, Chain, Base1, MOp1);
return DAG.getMergeValues(
- { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1),
- DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
- Load0.getValue(1), Load1.getValue(1)) }, dl);
+ {DAG.getNode(ISD::CONCAT_VECTORS, dl, MemN->getValueType(0), Load0,
+ Load1),
+ DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Load0.getValue(1),
+ Load1.getValue(1))},
+ dl);
}
if (MemOpc == ISD::STORE) {
assert(cast<StoreSDNode>(Op)->isUnindexed());
@@ -3497,27 +3516,35 @@ HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
if (MemOpc == ISD::MLOAD) {
VectorPair Thru =
opSplit(cast<MaskedLoadSDNode>(Op)->getPassThru(), dl, DAG);
- SDValue MLoad0 =
- DAG.getMaskedLoad(SingleTy, dl, Chain, Base0, Offset, Masks.first,
- Thru.first, SingleTy, MOp0, ISD::UNINDEXED,
- ISD::NON_EXTLOAD, false);
+ SDValue MLoad0 = DAG.getMaskedLoad(LoVT, dl, Chain, Base0, Offset,
+ Masks.first, Thru.first, LoMemVT, MOp0,
+ ISD::UNINDEXED, ISD::NON_EXTLOAD, false);
+
+ // The hi masked load has zero storage size. We therefore simply set it to
+ // the low masked load and rely on subsequent removal from the chain.
+ // See DAGTypeLegalizer::SplitVecRes_MLOAD().
SDValue MLoad1 =
- DAG.getMaskedLoad(SingleTy, dl, Chain, Base1, Offset, Masks.second,
- Thru.second, SingleTy, MOp1, ISD::UNINDEXED,
- ISD::NON_EXTLOAD, false);
+ HiIsEmpty ? MLoad0
+ : DAG.getMaskedLoad(HiVT, dl, Chain, Base1, Offset,
+ Masks.second, Thru.second, HiMemVT, MOp1,
+ ISD::UNINDEXED, ISD::NON_EXTLOAD, false);
return DAG.getMergeValues(
- { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, MLoad0, MLoad1),
- DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
- MLoad0.getValue(1), MLoad1.getValue(1)) }, dl);
+ {DAG.getNode(ISD::CONCAT_VECTORS, dl, MemN->getValueType(0), MLoad0,
+ MLoad1),
+ DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MLoad0.getValue(1),
+ MLoad1.getValue(1))},
+ dl);
}
if (MemOpc == ISD::MSTORE) {
VectorPair Vals = opSplit(cast<MaskedStoreSDNode>(Op)->getValue(), dl, DAG);
- SDValue MStore0 = DAG.getMaskedStore(Chain, dl, Vals.first, Base0, Offset,
- Masks.first, SingleTy, MOp0,
- ISD::UNINDEXED, false, false);
- SDValue MStore1 = DAG.getMaskedStore(Chain, dl, Vals.second, Base1, Offset,
- Masks.second, SingleTy, MOp1,
- ISD::UNINDEXED, false, false);
+ SDValue MStore0 =
+ DAG.getMaskedStore(Chain, dl, Vals.first, Base0, Offset, Masks.first,
+ LoMemVT, MOp0, ISD::UNINDEXED, false, false);
+ if (HiIsEmpty)
+ return MStore0;
+ SDValue MStore1 =
+ DAG.getMaskedStore(Chain, dl, Vals.second, Base1, Offset, Masks.second,
+ HiMemVT, MOp1, ISD::UNINDEXED, false, false);
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MStore0, MStore1);
}
More information about the llvm-commits
mailing list