[llvm] 27ec80f - [Hexagon] Avoid spurious high vmem for contained sub-HVX stores (#204661)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 23 01:23:34 PDT 2026
Author: L-roro
Date: 2026-07-23T10:23:30+02:00
New Revision: 27ec80f230b86f22bd7329d67965b48d41b18f92
URL: https://github.com/llvm/llvm-project/commit/27ec80f230b86f22bd7329d67965b48d41b18f92
DIFF: https://github.com/llvm/llvm-project/commit/27ec80f230b86f22bd7329d67965b48d41b18f92.diff
LOG: [Hexagon] Avoid spurious high vmem for contained sub-HVX stores (#204661)
When a sub-HVX store is widened into a masked HVX store, the unaligned
masked store lowering can split it into two vector stores: one at Base
and one at Base+HwLen. For stores whose original memory size fits within
the guaranteed alignment, the high half predicate is known to be all
false, so the Base+HwLen store is unnecessary.
Even an all-false predicated vmem can still probe the TLB. Emitting the
high store can therefore fault when Base+HwLen is on an unmapped page,
even though no bytes should be written there.
Preserve the original memory VT when widening sub-HVX stores, and use
that size during masked-store lowering to elide the empty high vmem.
Relevant links:
https://docs.qualcomm.com/doc/80-N2040-60/topic/conditional-execution.html#consuming-scalar-predicates
and
https://docs.qualcomm.com/doc/80-N2040-61/topic/memory.html#permissions
Co-authored-by: L-roro <rodriguez at roofline.ai>
Added:
llvm/test/CodeGen/Hexagon/autohvx/hvx-half-store-no-spurious-vmem.ll
Modified:
llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 22fe067efa6d9..54306e1c982c7 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -2566,10 +2566,23 @@ HexagonTargetLowering::LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const {
SDValue StoreLo =
getInstr(StoreOpc, dl, MVT::Other,
{MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG);
+ DAG.setNodeMemRefs(cast<MachineSDNode>(StoreLo.getNode()), {MemOp});
+
+ // If the store fits within one HwLen-aligned block, the high half's predicate
+ // is always all-zeros and the vmem(Base+HwLen) can be elided entirely.
+ // Proof: addr % StoreAlign == 0 and StoreMemSize <= StoreAlign implies
+ // addr % HwLen <= HwLen - StoreAlign, so addr % HwLen + StoreMemSize
+ // <= HwLen.
+ // Without this guard, Hexagon v73+ probes the TLB for vmem(Base+HwLen) even
+ // when the predicate is all-zeros, causing a TLBMISS if that page is
+ // unmapped.
+ uint64_t StoreMemSize = MaskN->getMemoryVT().getStoreSize().getFixedValue();
+ if (StoreMemSize <= MaskN->getAlign().value())
+ return StoreLo;
+
SDValue StoreHi =
getInstr(StoreOpc, dl, MVT::Other,
{MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG);
- DAG.setNodeMemRefs(cast<MachineSDNode>(StoreLo.getNode()), {MemOp});
DAG.setNodeMemRefs(cast<MachineSDNode>(StoreHi.getNode()), {MemOp});
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, {StoreLo, StoreHi});
}
@@ -3648,8 +3661,9 @@ HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const {
{DAG.getConstant(ValueLen, dl, MVT::i32)}, DAG);
MachineFunction &MF = DAG.getMachineFunction();
auto *MemOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen);
- return DAG.getMaskedStore(Chain, dl, Value, Base, Offset, Mask, ty(Value),
- MemOp, ISD::UNINDEXED, false, false);
+ return DAG.getMaskedStore(Chain, dl, Value, Base, Offset, Mask,
+ StoreN->getMemoryVT(), MemOp, ISD::UNINDEXED, false,
+ false);
}
SDValue
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/hvx-half-store-no-spurious-vmem.ll b/llvm/test/CodeGen/Hexagon/autohvx/hvx-half-store-no-spurious-vmem.ll
new file mode 100644
index 0000000000000..0681c06b834bd
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/hvx-half-store-no-spurious-vmem.ll
@@ -0,0 +1,44 @@
+; Regression test for: store <32 x half> at align 64 must NOT emit vmem(Base+HwLen).
+;
+; A <32 x half> store (64 bytes) at 64-byte alignment is widened by TypeWidenVector
+; to <64 x half> with mask [true*32, false*32]. LowerHvxMaskedOp's unaligned path used
+; to emit both:
+; if (q_lo) vmem(r+#0) = v_lo ; correct write
+; if (q_hi) vmem(r+#1) = v_hi ; spurious: q_hi is always all-zeros, but Hexagon v73
+; ; still probes TLB at (Base+128). TLBMISS if unmapped.
+;
+; With the fix (StoreMemSize <= StoreAlign), the high vmem is elided.
+;
+; RUN: llc -mtriple=hexagon -mattr=+hvxv73,+hvx-length128b -O2 < %s | FileCheck %s
+
+; CHECK-LABEL: half_vec_store_align64:
+; Store the lower half-vector (the real data) — must be present.
+; CHECK: if ({{q[0-9]+}}) vmem(r{{[0-9]+}}+#0) = v{{[0-9]+}}
+; There must be NO store to r+#1 (the spurious TLB-probing vmem).
+; CHECK-NOT: vmem(r{{[0-9]+}}+#1)
+
+; CHECK-LABEL: half_vec_store_align32:
+; CHECK-DAG: if ({{q[0-9]+}}) vmem(r{{[0-9]+}}+#0) = v{{[0-9]+}}
+; CHECK-DAG: if ({{q[0-9]+}}) vmem(r{{[0-9]+}}+#1) = v{{[0-9]+}}
+
+target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"
+target triple = "hexagon-unknown-elf"
+
+; Store 32 halfs into %out (64-byte aligned, NOT 128-byte aligned).
+; The store [%out .. %out+64) fits entirely within one HVX vector slot.
+; The spurious vmem at %out+128 must be absent.
+define void @half_vec_store_align64(ptr %out, ptr %in) {
+ %floats = load <32 x float>, ptr %in, align 128
+ %halfs = fptrunc <32 x float> %floats to <32 x half>
+ store <32 x half> %halfs, ptr %out, align 64
+ ret void
+}
+
+; With only 32-byte alignment, a 64-byte store may cross an HVX vector boundary.
+; The high vmem at %out+128 must still be emitted.
+define void @half_vec_store_align32(ptr %out, ptr %in) {
+ %floats = load <32 x float>, ptr %in, align 128
+ %halfs = fptrunc <32 x float> %floats to <32 x half>
+ store <32 x half> %halfs, ptr %out, align 32
+ ret void
+}
More information about the llvm-commits
mailing list