[llvm] DAG: Use poison for some load/store offsets in legalizer (PR #167756)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 05:03:50 PDT 2026
https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/167756
>From 39f7ff302a288561bd1bf142c00d89af618c2b12 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 12 Nov 2025 12:18:03 -0800
Subject: [PATCH] DAG: Use poison for some load/store offsets in legalizer
Unindexed load/store offsets are a don't-care operand that must be poison
rather than undef now that poison is legal. Convert the remaining producers
that build the offset with getUNDEF (in SelectionDAGBuilder and the ARM,
Hexagon, RISCV, and X86 lowerings) to getPOISON, and strengthen the offset
asserts in the SelectionDAG memory-node builders to require POISON exactly
instead of accepting any undef-or-poison value.
Co-Authored-By: Claude (Claude-Opus-4.8) <noreply at anthropic.com>
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 39 ++++++++++++-------
.../SelectionDAG/SelectionDAGBuilder.cpp | 8 ++--
llvm/lib/Target/ARM/ARMISelLowering.cpp | 4 +-
.../Target/Hexagon/HexagonISelLoweringHVX.cpp | 6 +--
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 13 +++----
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
6 files changed, 41 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 405ae80671d71..683cf2517b2f5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -10710,7 +10710,8 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
"Range metadata and load type must match!");
bool Indexed = AM != ISD::UNINDEXED;
- assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
+ assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
+ "Unindexed load with an offset!");
SDVTList VTs = Indexed ?
getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
@@ -10779,7 +10780,8 @@ SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
SDValue Base, SDValue Offset,
ISD::MemIndexedMode AM) {
LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
- assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
+ assert(LD->getOffset().getOpcode() == ISD::POISON &&
+ "Load is already a indexed load!");
// Don't propagate the invariant or dereferenceable flags.
auto MMOFlags =
LD->getMemOperand()->getFlags() &
@@ -10838,7 +10840,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
}
bool Indexed = AM != ISD::UNINDEXED;
- assert((Indexed || Offset.isUndef()) && "Unindexed store with an offset!");
+ assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
+ "Unindexed store with an offset!");
SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
: getVTList(MVT::Other);
SDValue Ops[] = {Chain, Val, Ptr, Offset};
@@ -10912,7 +10915,8 @@ SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
SDValue Base, SDValue Offset,
ISD::MemIndexedMode AM) {
StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
- assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
+ assert(ST->getOffset().getOpcode() == ISD::POISON &&
+ "Store is already a indexed store!");
return getStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
ST->getMemoryVT(), ST->getMemOperand(), AM,
ST->isTruncatingStore());
@@ -10951,7 +10955,8 @@ SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
"Vector width mismatch between mask and data");
bool Indexed = AM != ISD::UNINDEXED;
- assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
+ assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
+ "Unindexed load with an offset!");
SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
: getVTList(VT, MVT::Other);
@@ -11027,7 +11032,8 @@ SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
SDValue Base, SDValue Offset,
ISD::MemIndexedMode AM) {
auto *LD = cast<VPLoadSDNode>(OrigLoad);
- assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
+ assert(LD->getOffset().getOpcode() == ISD::POISON &&
+ "Load is already a indexed load!");
// Don't propagate the invariant or dereferenceable flags.
auto MMOFlags =
LD->getMemOperand()->getFlags() &
@@ -11050,7 +11056,8 @@ SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
"Vector width mismatch between mask and data");
bool Indexed = AM != ISD::UNINDEXED;
- assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
+ assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
+ "Unindexed vp_store with an offset!");
SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
: getVTList(MVT::Other);
SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
@@ -11152,7 +11159,8 @@ SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
SDValue Base, SDValue Offset,
ISD::MemIndexedMode AM) {
auto *ST = cast<VPStoreSDNode>(OrigStore);
- assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
+ assert(ST->getOffset().getOpcode() == ISD::POISON &&
+ "Store is already an indexed store!");
SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
Offset, ST->getMask(), ST->getVectorLength()};
@@ -11183,7 +11191,8 @@ SDValue SelectionDAG::getStridedLoadVP(
SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
bool Indexed = AM != ISD::UNINDEXED;
- assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
+ assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
+ "Unindexed load with an offset!");
SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
@@ -11240,7 +11249,8 @@ SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
bool IsTruncating, bool IsCompressing) {
assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
bool Indexed = AM != ISD::UNINDEXED;
- assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
+ assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
+ "Unindexed vp_store with an offset!");
SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
: getVTList(MVT::Other);
SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
@@ -11410,7 +11420,7 @@ SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
ISD::MemIndexedMode AM,
ISD::LoadExtType ExtTy, bool isExpanding) {
bool Indexed = AM != ISD::UNINDEXED;
- assert((Indexed || Offset.isUndef()) &&
+ assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
"Unindexed masked load with an offset!");
SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
: getVTList(VT, MVT::Other);
@@ -11442,7 +11452,8 @@ SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
SDValue Base, SDValue Offset,
ISD::MemIndexedMode AM) {
MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
- assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
+ assert(LD->getOffset().getOpcode() == ISD::POISON &&
+ "Masked load is already a indexed load!");
return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
Offset, LD->getMask(), LD->getPassThru(),
LD->getMemoryVT(), LD->getMemOperand(), AM,
@@ -11458,7 +11469,7 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
bool Indexed = AM != ISD::UNINDEXED;
- assert((Indexed || Offset.isUndef()) &&
+ assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
"Unindexed masked store with an offset!");
SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
: getVTList(MVT::Other);
@@ -11491,7 +11502,7 @@ SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
SDValue Base, SDValue Offset,
ISD::MemIndexedMode AM) {
MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
- assert(ST->getOffset().isUndef() &&
+ assert(ST->getOffset().getOpcode() == ISD::POISON &&
"Masked store is already a indexed store!");
return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f1c9809c2159e..385f3729d46c8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4993,7 +4993,7 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
SDValue Ptr = getValue(PtrOperand);
SDValue Src0 = getValue(Src0Operand);
SDValue Mask = getValue(MaskOperand);
- SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
+ SDValue Offset = DAG.getPOISON(Ptr.getValueType());
EVT VT = Src0.getValueType();
@@ -5143,7 +5143,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
SDValue Ptr = getValue(PtrOperand);
SDValue Src0 = getValue(Src0Operand);
SDValue Mask = getValue(MaskOperand);
- SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
+ SDValue Offset = DAG.getPOISON(Ptr.getValueType());
EVT VT = Src0.getValueType();
AAMDNodes AAInfo = I.getAAMetadata();
@@ -8915,7 +8915,7 @@ void SelectionDAGBuilder::visitVPStore(
if (!Alignment)
Alignment = DAG.getEVTAlign(VT);
SDValue Ptr = OpValues[1];
- SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
+ SDValue Offset = DAG.getPOISON(Ptr.getValueType());
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
MachineMemOperand::Flags MMOFlags =
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
@@ -9019,7 +9019,7 @@ void SelectionDAGBuilder::visitVPStridedStore(
SDValue ST = DAG.getStridedStoreVP(
getMemoryRoot(), DL, OpValues[0], OpValues[1],
- DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
+ DAG.getPOISON(OpValues[1].getValueType()), OpValues[2], OpValues[3],
OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
/*IsCompressing*/ false);
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index a88cea970e9e9..e2b39726c7c0d 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -18030,7 +18030,7 @@ static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
ISD::LoadExtType NewExtType =
N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
- SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
+ SDValue Offset = DAG.getPOISON(BasePtr.getValueType());
EVT NewFromVT = EVT::getVectorVT(
C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements);
EVT NewToVT = EVT::getVectorVT(
@@ -19002,7 +19002,7 @@ static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N,
MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
AAMDNodes AAInfo = LD->getAAInfo();
- SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
+ SDValue Offset = DAG.getPOISON(BasePtr.getValueType());
EVT NewFromVT = EVT::getVectorVT(
C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements);
EVT NewToVT = EVT::getVectorVT(
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 3be35ab3482d6..670fe270f36a1 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -3568,7 +3568,7 @@ HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
auto MaskN = cast<MaskedLoadStoreSDNode>(Op);
assert(MaskN->isUnindexed());
VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG);
- SDValue Offset = DAG.getUNDEF(MVT::i32);
+ SDValue Offset = DAG.getPOISON(MVT::i32);
if (MemOpc == ISD::MLOAD) {
VectorPair Thru =
@@ -3619,7 +3619,7 @@ HexagonTargetLowering::WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const {
SDValue Chain = LoadN->getChain();
SDValue Base = LoadN->getBasePtr();
- SDValue Offset = DAG.getUNDEF(MVT::i32);
+ SDValue Offset = DAG.getPOISON(MVT::i32);
MVT ResTy = ty(Op);
unsigned HwLen = Subtarget.getVectorLength();
@@ -3651,7 +3651,7 @@ HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const {
SDValue Chain = StoreN->getChain();
SDValue Base = StoreN->getBasePtr();
- SDValue Offset = DAG.getUNDEF(MVT::i32);
+ SDValue Offset = DAG.getPOISON(MVT::i32);
SDValue Value = opCastElem(StoreN->getValue(), MVT::i8, DAG);
MVT ValueTy = ty(Value);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index fe905f686d855..0e48270b1c194 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -7596,7 +7596,7 @@ SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
return DAG.getStoreVP(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
- DAG.getUNDEF(Store->getBasePtr().getValueType()),
+ DAG.getPOISON(Store->getBasePtr().getValueType()),
DAG.getAllOnesConstant(DL, MaskVT), VL, NewVT, MMO,
ISD::UNINDEXED);
}
@@ -23304,11 +23304,10 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
SmallVector<int> ShuffleMask;
if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
matchIndexAsShuffle(VT, Index, MGN->getMask(), ShuffleMask)) {
- SDValue Load = DAG.getMaskedLoad(VT, DL, MGN->getChain(),
- MGN->getBasePtr(), DAG.getUNDEF(XLenVT),
- MGN->getMask(), DAG.getUNDEF(VT),
- MGN->getMemoryVT(), MGN->getMemOperand(),
- ISD::UNINDEXED, ISD::NON_EXTLOAD);
+ SDValue Load = DAG.getMaskedLoad(
+ VT, DL, MGN->getChain(), MGN->getBasePtr(), DAG.getPOISON(XLenVT),
+ MGN->getMask(), DAG.getPOISON(VT), MGN->getMemoryVT(),
+ MGN->getMemOperand(), ISD::UNINDEXED, ISD::NON_EXTLOAD);
SDValue Shuffle =
DAG.getVectorShuffle(VT, DL, Load, DAG.getUNDEF(VT), ShuffleMask);
return DAG.getMergeValues({Shuffle, Load.getValue(1)}, DL);
@@ -23375,7 +23374,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
SDValue Shuffle = DAG.getVectorShuffle(VT, DL, MSN->getValue(),
DAG.getUNDEF(VT), ShuffleMask);
return DAG.getMaskedStore(MSN->getChain(), DL, Shuffle, MSN->getBasePtr(),
- DAG.getUNDEF(XLenVT), MSN->getMask(),
+ DAG.getPOISON(XLenVT), MSN->getMask(),
MSN->getMemoryVT(), MSN->getMemOperand(),
ISD::UNINDEXED, false);
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1de4ffa33e960..22715aa870d7c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -28582,7 +28582,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
- SDValue Offset = DAG.getUNDEF(VMask.getValueType());
+ SDValue Offset = DAG.getPOISON(VMask.getValueType());
return DAG.getMaskedStore(Chain, dl, DataToTruncate, Addr, Offset, VMask,
MemVT, MemIntr->getMemOperand(), ISD::UNINDEXED,
More information about the llvm-commits
mailing list