[llvm] [CodeGen] Improve `getLoadExtAction` and friends (PR #181104)
Demetrius Kanios via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 12 00:41:31 PST 2026
https://github.com/QuantumSegfault created https://github.com/llvm/llvm-project/pull/181104
Alternative approach to the same goals as #162407
This takes `TargetLoweringBase::getLoadExtAction`, makes it virtual, merges `getAtomicLoadExtAction` into it, and adds more inputs for relavent information (alignment, MMOFlags, address space).
This is fully backwards compatible, with the existing setLoadExtAction working as before. But this allows targets to override the query to make more use of the information.
>From f84c6cf62a45b3450e1b68b0c5019aa2c5db0dcf Mon Sep 17 00:00:00 2001
From: Demetrius Kanios <demetrius at kanios.net>
Date: Thu, 12 Feb 2026 00:26:21 -0800
Subject: [PATCH] Collapse `getLoadExtAction` and `getAtomicLoadExtAction` into
single, richer `getLoadAction`
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 44 ++-
llvm/include/llvm/CodeGen/TargetLowering.h | 59 ++--
llvm/lib/CodeGen/CodeGenPrepare.cpp | 14 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 292 +++++++++++-------
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 32 +-
.../SelectionDAG/LegalizeVectorOps.cpp | 4 +-
.../CodeGen/SelectionDAG/TargetLowering.cpp | 9 +-
llvm/lib/CodeGen/TargetLoweringBase.cpp | 23 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 4 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 4 +-
10 files changed, 315 insertions(+), 170 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 21afcbefdf719..132dc9d18845d 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -28,6 +28,7 @@
#include "llvm/Analysis/TargetTransformInfoImpl.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
@@ -1261,10 +1262,41 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
EVT ExtVT = EVT::getEVT(Dst);
EVT LoadVT = EVT::getEVT(Src);
unsigned LType =
- ((Opcode == Instruction::ZExt) ? ISD::ZEXTLOAD : ISD::SEXTLOAD);
- if (DstLT.first == SrcLT.first &&
- TLI->isLoadExtLegal(LType, ExtVT, LoadVT))
- return 0;
+ ((Opcode == Instruction::ZExt) ? ISD::ZEXTLOAD : ISD::SEXTLOAD);
+ if (I) {
+ if (auto *LI = dyn_cast<LoadInst>(I->getOperand(0))) {
+ if (DstLT.first == SrcLT.first &&
+ TLI->isLoadLegal(
+ ExtVT, LoadVT, LI->getAlign(),
+ TLI->getLoadMemOperandFlags(*LI, DL),
+ LI->getPointerAddressSpace(), LType, false))
+ return 0;
+ } else if (auto *II = dyn_cast<IntrinsicInst>(I->getOperand(0))) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::masked_load: {
+ Type *PtrType = II->getArgOperand(0)->getType();
+ assert(PtrType->isPointerTy());
+
+ auto MMOFlags = MachineMemOperand::MOLoad;
+ if (I->hasMetadata(LLVMContext::MD_nontemporal))
+ MMOFlags |= MachineMemOperand::MONonTemporal;
+ if (I->hasMetadata(LLVMContext::MD_invariant_load))
+ MMOFlags |= MachineMemOperand::MOInvariant;
+
+ if (DstLT.first == SrcLT.first &&
+ TLI->isLoadLegal(ExtVT, LoadVT,
+ II->getParamAlign(0).valueOrOne(), MMOFlags,
+ PtrType->getPointerAddressSpace(), LType,
+ false))
+ return 0;
+
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ }
}
break;
case Instruction::AddrSpaceCast:
@@ -1556,7 +1588,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
if (Opcode == Instruction::Store)
LA = getTLI()->getTruncStoreAction(LT.second, MemVT);
else
- LA = getTLI()->getLoadExtAction(ISD::EXTLOAD, LT.second, MemVT);
+ LA = getTLI()->getLoadAction(LT.second, MemVT, Alignment,
+ MachineMemOperand::Flags::MOLoad,
+ AddressSpace, ISD::EXTLOAD, false);
if (LA != TargetLowering::Legal && LA != TargetLowering::Custom) {
// This is a vector load/store for some illegal type that is scalarized.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index ada4ffd3bcc89..1bdf87e92edcc 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -31,6 +31,7 @@
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/LibcallLoweringInfo.h"
#include "llvm/CodeGen/LowLevelTypeUtils.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcallUtil.h"
#include "llvm/CodeGen/SelectionDAG.h"
@@ -1511,49 +1512,29 @@ class LLVM_ABI TargetLoweringBase {
/// Return how this load with extension should be treated: either it is legal,
/// needs to be promoted to a larger size, needs to be expanded to some other
/// code sequence, or the target has a custom expander for it.
- LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT,
- EVT MemVT) const {
- if (ValVT.isExtended() || MemVT.isExtended()) return Expand;
- unsigned ValI = (unsigned) ValVT.getSimpleVT().SimpleTy;
- unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy;
- assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::VALUETYPE_SIZE &&
- MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!");
- unsigned Shift = 4 * ExtType;
- return (LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf);
- }
+ virtual LegalizeAction getLoadAction(EVT ValVT, EVT MemVT, Align Alignment,
+ MachineMemOperand::Flags MMOFlags,
+ unsigned AddrSpace, unsigned ExtType,
+ bool Atomic) const;
/// Return true if the specified load with extension is legal on this target.
- bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const {
- return getLoadExtAction(ExtType, ValVT, MemVT) == Legal;
+ bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment,
+ MachineMemOperand::Flags MMOFlags, unsigned AddrSpace,
+ unsigned ExtType, bool Atomic) const {
+ return getLoadAction(ValVT, MemVT, Alignment, MMOFlags, AddrSpace, ExtType,
+ Atomic) == Legal;
}
/// Return true if the specified load with extension is legal or custom
/// on this target.
- bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const {
- return getLoadExtAction(ExtType, ValVT, MemVT) == Legal ||
- getLoadExtAction(ExtType, ValVT, MemVT) == Custom;
- }
-
- /// Same as getLoadExtAction, but for atomic loads.
- LegalizeAction getAtomicLoadExtAction(unsigned ExtType, EVT ValVT,
- EVT MemVT) const {
- if (ValVT.isExtended() || MemVT.isExtended()) return Expand;
- unsigned ValI = (unsigned)ValVT.getSimpleVT().SimpleTy;
- unsigned MemI = (unsigned)MemVT.getSimpleVT().SimpleTy;
- assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::VALUETYPE_SIZE &&
- MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!");
- unsigned Shift = 4 * ExtType;
- LegalizeAction Action =
- (LegalizeAction)((AtomicLoadExtActions[ValI][MemI] >> Shift) & 0xf);
- assert((Action == Legal || Action == Expand) &&
- "Unsupported atomic load extension action.");
- return Action;
- }
-
- /// Return true if the specified atomic load with extension is legal on
- /// this target.
- bool isAtomicLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const {
- return getAtomicLoadExtAction(ExtType, ValVT, MemVT) == Legal;
+ bool isLoadLegalOrCustom(EVT ValVT, EVT MemVT, Align Alignment,
+ MachineMemOperand::Flags MMOFlags,
+ unsigned AddrSpace, unsigned ExtType,
+ bool Atomic) const {
+ return getLoadAction(ValVT, MemVT, Alignment, MMOFlags, AddrSpace, ExtType,
+ Atomic) == Legal ||
+ getLoadAction(ValVT, MemVT, Alignment, MMOFlags, AddrSpace, ExtType,
+ Atomic) == Custom;
}
/// Return how this store with truncation should be treated: either it is
@@ -3182,7 +3163,9 @@ class LLVM_ABI TargetLoweringBase {
LType = ISD::SEXTLOAD;
}
- return isLoadExtLegal(LType, VT, LoadVT);
+ return isLoadLegal(VT, LoadVT, Load->getAlign(),
+ getLoadMemOperandFlags(*Load, DL),
+ Load->getPointerAddressSpace(), LType, false);
}
/// Return true if any actual instruction that defines a value of type FromTy
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index bf56ffd3b4b7b..c792e6d0e0574 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7495,12 +7495,12 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
uint32_t ActiveBits = DemandBits.getActiveBits();
// Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the
- // target even if isLoadExtLegal says an i1 EXTLOAD is valid. For example,
- // for the AArch64 target isLoadExtLegal(ZEXTLOAD, i32, i1) returns true, but
- // (and (load x) 1) is not matched as a single instruction, rather as a LDR
- // followed by an AND.
+ // target even if isLoadLegal says an i1 EXTLOAD is valid. For example,
+ // for the AArch64 target isLoadLegal(i32, i1, ..., ZEXTLOAD, false) returns
+ // true, but (and (load x) 1) is not matched as a single instruction, rather
+ // as a LDR followed by an AND.
// TODO: Look into removing this restriction by fixing backends to either
- // return false for isLoadExtLegal for i1 or have them select this pattern to
+ // return false for isLoadLegal for i1 or have them select this pattern to
// a single instruction.
//
// Also avoid hoisting if we didn't see any ands with the exact DemandBits
@@ -7515,7 +7515,9 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
// Reject cases that won't be matched as extloads.
if (!LoadResultVT.bitsGT(TruncVT) || !TruncVT.isRound() ||
- !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT))
+ !TLI->isLoadLegal(LoadResultVT, TruncVT, Load->getAlign(),
+ TLI->getLoadMemOperandFlags(*Load, *DL),
+ Load->getPointerAddressSpace(), ISD::ZEXTLOAD, false))
return false;
IRBuilder<> Builder(Load->getNextNode());
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 58c59628342c7..b2f36d4074cdf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7047,7 +7047,9 @@ bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
if (ExtVT == LoadedVT &&
(!LegalOperations ||
- TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
+ TLI.isLoadLegal(LoadResultTy, ExtVT, LoadN->getAlign(),
+ LoadN->getMemOperand()->getFlags(),
+ LoadN->getAddressSpace(), ISD::ZEXTLOAD, false))) {
// ZEXTLOAD will match without needing to change the size of the value being
// loaded.
return true;
@@ -7063,7 +7065,9 @@ bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
return false;
if (LegalOperations &&
- !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
+ !TLI.isLoadLegal(LoadResultTy, ExtVT, LoadN->getAlign(),
+ LoadN->getMemOperand()->getFlags(),
+ LoadN->getAddressSpace(), ISD::ZEXTLOAD, false))
return false;
if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT, /*ByteOffset=*/0))
@@ -7126,7 +7130,9 @@ bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
return false;
if (LegalOperations &&
- !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT))
+ !TLI.isLoadLegal(Load->getValueType(0), MemVT, Load->getAlign(),
+ Load->getMemOperand()->getFlags(),
+ Load->getAddressSpace(), ExtType, false))
return false;
// For the transform to be legal, the load must produce only two values
@@ -7638,7 +7644,9 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
ConstantSDNode *Splat = isConstOrConstSplat(N1, true, true);
if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && Splat) {
EVT MemVT = MLoad->getMemoryVT();
- if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT)) {
+ if (TLI.isLoadLegal(VT, MemVT, MLoad->getAlign(),
+ MLoad->getMemOperand()->getFlags(),
+ MLoad->getAddressSpace(), ISD::ZEXTLOAD, false)) {
// For this AND to be a zero extension of the masked load the elements
// of the BuildVec must mask the bottom bits of the extended element
// type
@@ -7785,9 +7793,10 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
// If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
// actually legal and isn't going to get expanded, else this is a false
// optimisation.
- bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
- Load->getValueType(0),
- Load->getMemoryVT());
+ bool CanZextLoadProfitably =
+ TLI.isLoadLegal(Load->getValueType(0), Load->getMemoryVT(),
+ Load->getAlign(), Load->getMemOperand()->getFlags(),
+ Load->getAddressSpace(), ISD::ZEXTLOAD, false);
// Resize the constant to the same size as the original memory access before
// extension. If it is still the AllOnesValue then this AND is completely
@@ -7979,7 +7988,9 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
APInt ExtBits = APInt::getHighBitsSet(ExtBitSize, ExtBitSize - MemBitSize);
if (DAG.MaskedValueIsZero(N1, ExtBits) &&
((!LegalOperations && LN0->isSimple()) ||
- TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
+ TLI.isLoadLegal(VT, MemVT, LN0->getAlign(),
+ LN0->getMemOperand()->getFlags(),
+ LN0->getAddressSpace(), ISD::ZEXTLOAD, false))) {
SDValue ExtLoad =
DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(),
LN0->getBasePtr(), MemVT, LN0->getMemOperand());
@@ -9901,10 +9912,17 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
// Before legalize we can introduce too wide illegal loads which will be later
// split into legal sized loads. This enables us to combine i64 load by i8
// patterns to a couple of i32 loads on 32 bit targets.
- if (LegalOperations &&
- !TLI.isLoadExtLegal(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, VT,
- MemVT))
- return SDValue();
+ if (LegalOperations) {
+ for (auto *L : Loads) {
+ if (!TLI.isLoadLegal(VT, MemVT, L->getAlign(),
+ L->getMemOperand()->getFlags(), L->getAddressSpace(),
+ NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD,
+ false)) {
+
+ return SDValue();
+ }
+ }
+ }
// Check if the bytes of the OR we are looking at match with either big or
// little endian value load
@@ -13782,20 +13800,27 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) {
unsigned WideWidth = WideVT.getScalarSizeInBits();
bool IsSigned = isSignedIntSetCC(CC);
auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
- if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() &&
- SetCCWidth != 1 && SetCCWidth < WideWidth &&
- TLI.isLoadExtLegalOrCustom(LoadExtOpcode, WideVT, NarrowVT) &&
+ if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && SetCCWidth != 1 &&
+ SetCCWidth < WideWidth &&
TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) {
- // Both compare operands can be widened for free. The LHS can use an
- // extended load, and the RHS is a constant:
- // vselect (ext (setcc load(X), C)), N1, N2 -->
- // vselect (setcc extload(X), C'), N1, N2
- auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
- SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
- SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
- EVT WideSetCCVT = getSetCCResultType(WideVT);
- SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
- return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
+ LoadSDNode *Ld = cast<LoadSDNode>(LHS);
+
+ if (TLI.isLoadLegalOrCustom(WideVT, NarrowVT, Ld->getAlign(),
+ Ld->getMemOperand()->getFlags(),
+ Ld->getAddressSpace(), LoadExtOpcode,
+ false)) {
+ // Both compare operands can be widened for free. The LHS can use an
+ // extended load, and the RHS is a constant:
+ // vselect (ext (setcc load(X), C)), N1, N2 -->
+ // vselect (setcc extload(X), C'), N1, N2
+ auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+ SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
+ SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
+ EVT WideSetCCVT = getSetCCResultType(WideVT);
+ SDValue WideSetCC =
+ DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
+ return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
+ }
}
}
@@ -14230,8 +14255,12 @@ static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI,
// Combine2), so we should conservatively check the OperationAction.
LoadSDNode *Load1 = cast<LoadSDNode>(Op1);
LoadSDNode *Load2 = cast<LoadSDNode>(Op2);
- if (!TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load1->getMemoryVT()) ||
- !TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load2->getMemoryVT()) ||
+ if (!TLI.isLoadLegal(VT, Load1->getMemoryVT(), Load1->getAlign(),
+ Load1->getMemOperand()->getFlags(),
+ Load1->getAddressSpace(), ExtLoadOpcode, false) ||
+ !TLI.isLoadLegal(VT, Load2->getMemoryVT(), Load2->getAlign(),
+ Load2->getMemOperand()->getFlags(),
+ Load2->getAddressSpace(), ExtLoadOpcode, false) ||
(N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes &&
TLI.getOperationAction(ISD::VSELECT, VT) != TargetLowering::Legal))
return SDValue();
@@ -14455,13 +14484,17 @@ SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
// Try to split the vector types to get down to legal types.
EVT SplitSrcVT = SrcVT;
EVT SplitDstVT = DstVT;
- while (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT) &&
+ while (!TLI.isLoadLegalOrCustom(SplitDstVT, SplitSrcVT, LN0->getAlign(),
+ LN0->getMemOperand()->getFlags(),
+ LN0->getAddressSpace(), ExtType, false) &&
SplitSrcVT.getVectorNumElements() > 1) {
SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
}
- if (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT))
+ if (!TLI.isLoadLegalOrCustom(SplitDstVT, SplitSrcVT, LN0->getAlign(),
+ LN0->getMemOperand()->getFlags(),
+ LN0->getAddressSpace(), ExtType, false))
return SDValue();
assert(!DstVT.isScalableVector() && "Unexpected scalable vector type");
@@ -14534,7 +14567,9 @@ SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
return SDValue();
LoadSDNode *Load = cast<LoadSDNode>(N1.getOperand(0));
EVT MemVT = Load->getMemoryVT();
- if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) ||
+ if (!TLI.isLoadLegal(VT, MemVT, Load->getAlign(),
+ Load->getMemOperand()->getFlags(),
+ Load->getAddressSpace(), ISD::ZEXTLOAD, false) ||
Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed())
return SDValue();
@@ -14647,7 +14682,9 @@ static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
EVT MemVT = OldExtLoad->getMemoryVT();
if ((LegalOperations || !OldExtLoad->isSimple() || VT.isVector()) &&
- !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT))
+ !TLI.isLoadLegal(VT, MemVT, OldExtLoad->getAlign(),
+ OldExtLoad->getMemOperand()->getFlags(),
+ OldExtLoad->getAddressSpace(), ExtLoadType, false))
return SDValue();
SDLoc DL(OldExtLoad);
@@ -14707,7 +14744,9 @@ static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
// code generation being the result of that target's implementation of
// isVectorLoadExtDesirable().
if ((LegalOperations || VT.isFixedLengthVector() || !Load->isSimple()) &&
- !TLI.isLoadExtLegal(ExtLoadType, VT, Load->getValueType(0)))
+ !TLI.isLoadLegal(VT, Load->getValueType(0), Load->getAlign(),
+ Load->getMemOperand()->getFlags(),
+ Load->getAddressSpace(), ExtLoadType, false))
return {};
bool DoXform = true;
@@ -14763,7 +14802,9 @@ tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT,
return SDValue();
if ((LegalOperations || !cast<MaskedLoadSDNode>(N0)->isSimple()) &&
- !TLI.isLoadExtLegalOrCustom(ExtLoadType, VT, Ld->getValueType(0)))
+ !TLI.isLoadLegalOrCustom(VT, Ld->getValueType(0), Ld->getAlign(),
+ Ld->getMemOperand()->getFlags(),
+ Ld->getAddressSpace(), ExtLoadType, false))
return SDValue();
if (!TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
@@ -14788,7 +14829,7 @@ static SDValue tryToFoldExtOfAtomicLoad(SelectionDAG &DAG,
if (!ALoad || ALoad->getOpcode() != ISD::ATOMIC_LOAD)
return {};
EVT MemoryVT = ALoad->getMemoryVT();
- if (!TLI.isAtomicLoadExtLegal(ExtLoadType, VT, MemoryVT))
+ if (!TLI.isLoadLegal(VT, MemoryVT, ALoad->getAlign(), ALoad->getMemOperand()->getFlags(), ALoad->getAddressSpace(), ExtLoadType, true))
return {};
// Can't fold into ALoad if it is already extending differently.
ISD::LoadExtType ALoadExtTy = ALoad->getExtensionType();
@@ -14905,9 +14946,15 @@ SDValue DAGCombiner::foldSextSetcc(SDNode *N) {
// legal {z/s}ext-load.
// TODO: Allow widening of an existing {z/s}ext-load?
if (!(ISD::isNON_EXTLoad(V.getNode()) &&
- ISD::isUNINDEXEDLoad(V.getNode()) &&
- cast<LoadSDNode>(V)->isSimple() &&
- TLI.isLoadExtLegal(LoadOpcode, VT, V.getValueType())))
+ ISD::isUNINDEXEDLoad(V.getNode())))
+ return false;
+
+ LoadSDNode *Ld = cast<LoadSDNode>(V.getNode());
+
+ if (!(Ld->isSimple() &&
+ TLI.isLoadLegal(VT, V.getValueType(), Ld->getAlign(),
+ Ld->getMemOperand()->getFlags(),
+ Ld->getAddressSpace(), LoadOpcode, false)))
return false;
// Non-chain users of this value must either be the setcc in this
@@ -15104,8 +15151,10 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
(!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
EVT MemVT = LN00->getMemoryVT();
- if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) &&
- LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) {
+ if (TLI.isLoadLegal(VT, MemVT, LN00->getAlign(),
+ LN00->getMemOperand()->getFlags(),
+ LN00->getAddressSpace(), ISD::SEXTLOAD, false) &&
+ LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) {
SmallVector<SDNode*, 4> SetCCs;
bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
ISD::SIGN_EXTEND, SetCCs, TLI);
@@ -15422,7 +15471,9 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
(!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
EVT MemVT = LN00->getMemoryVT();
- if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) &&
+ if (TLI.isLoadLegal(VT, MemVT, LN00->getAlign(),
+ LN00->getMemOperand()->getFlags(),
+ LN00->getAddressSpace(), ISD::ZEXTLOAD, false) &&
LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) {
bool DoXform = true;
SmallVector<SDNode*, 4> SetCCs;
@@ -15652,31 +15703,34 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
ISD::ZEXTLOAD, ISD::ZERO_EXTEND))
return foldedExt;
} else if (ISD::isNON_EXTLoad(N0.getNode()) &&
- ISD::isUNINDEXEDLoad(N0.getNode()) &&
- TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) {
- bool DoXform = true;
- SmallVector<SDNode *, 4> SetCCs;
- if (!N0.hasOneUse())
- DoXform =
- ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
- if (DoXform) {
- LoadSDNode *LN0 = cast<LoadSDNode>(N0);
- SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(),
- LN0->getBasePtr(), N0.getValueType(),
- LN0->getMemOperand());
- ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND);
- // If the load value is used only by N, replace it via CombineTo N.
- bool NoReplaceTrunc = N0.hasOneUse();
- CombineTo(N, ExtLoad);
- if (NoReplaceTrunc) {
- DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
- recursivelyDeleteUnusedNodes(LN0);
- } else {
- SDValue Trunc =
- DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
- CombineTo(LN0, Trunc, ExtLoad.getValue(1));
+ ISD::isUNINDEXEDLoad(N0.getNode())) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ if (TLI.isLoadLegalOrCustom(VT, N0.getValueType(), LN0->getAlign(),
+ LN0->getMemOperand()->getFlags(),
+ LN0->getAddressSpace(), ISD::EXTLOAD, false)) {
+ bool DoXform = true;
+ SmallVector<SDNode *, 4> SetCCs;
+ if (!N0.hasOneUse())
+ DoXform =
+ ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
+ if (DoXform) {
+ SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(),
+ LN0->getBasePtr(), N0.getValueType(),
+ LN0->getMemOperand());
+ ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND);
+ // If the load value is used only by N, replace it via CombineTo N.
+ bool NoReplaceTrunc = N0.hasOneUse();
+ CombineTo(N, ExtLoad);
+ if (NoReplaceTrunc) {
+ DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
+ recursivelyDeleteUnusedNodes(LN0);
+ } else {
+ SDValue Trunc =
+ DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
+ CombineTo(LN0, Trunc, ExtLoad.getValue(1));
+ }
+ return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
- return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
}
@@ -15688,7 +15742,10 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
LoadSDNode *LN0 = cast<LoadSDNode>(N0);
ISD::LoadExtType ExtType = LN0->getExtensionType();
EVT MemVT = LN0->getMemoryVT();
- if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) {
+ if (!LegalOperations ||
+ TLI.isLoadLegal(VT, MemVT, LN0->getAlign(),
+ LN0->getMemOperand()->getFlags(),
+ LN0->getAddressSpace(), ExtType, false)) {
SDValue ExtLoad =
DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), LN0->getBasePtr(),
MemVT, LN0->getMemOperand());
@@ -16002,7 +16059,9 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
EVT::getIntegerVT(*DAG.getContext(), ShiftMask.countr_one());
// If the mask is smaller, recompute the type.
if ((ExtVT.getScalarSizeInBits() > MaskedVT.getScalarSizeInBits()) &&
- TLI.isLoadExtLegal(ExtType, SRL.getValueType(), MaskedVT))
+ TLI.isLoadLegal(SRL.getValueType(), MaskedVT, LN->getAlign(),
+ LN->getMemOperand()->getFlags(),
+ LN->getAddressSpace(), ExtType, false))
ExtVT = MaskedVT;
} else if (ExtType == ISD::ZEXTLOAD &&
ShiftMask.isShiftedMask(Offset, ActiveBits) &&
@@ -16011,7 +16070,9 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
// If the mask is shifted we can use a narrower load and a shl to insert
// the trailing zeros.
if (((Offset + ActiveBits) <= ExtVT.getScalarSizeInBits()) &&
- TLI.isLoadExtLegal(ExtType, SRL.getValueType(), MaskedVT)) {
+ TLI.isLoadLegal(SRL.getValueType(), MaskedVT, LN->getAlign(),
+ LN->getMemOperand()->getFlags(),
+ LN->getAddressSpace(), ExtType, false)) {
ExtVT = MaskedVT;
ShAmt = Offset + ShAmt;
ShiftedOffset = Offset;
@@ -16233,33 +16294,39 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
// If sextload is not supported by target, we can only do the combine when
// load has one use. Doing otherwise can block folding the extload with other
// extends that the target does support.
- if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
- ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
- ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple() &&
- N0.hasOneUse()) ||
- TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) {
+ if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
auto *LN0 = cast<LoadSDNode>(N0);
- SDValue ExtLoad =
- DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
- LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
- CombineTo(N, ExtLoad);
- CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
- AddToWorklist(ExtLoad.getNode());
- return SDValue(N, 0); // Return N so it doesn't get rechecked!
+ if (ExtVT == LN0->getMemoryVT() &&
+ ((!LegalOperations && LN0->isSimple() && N0.hasOneUse()) ||
+ TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(),
+ LN0->getMemOperand()->getFlags(),
+ LN0->getAddressSpace(), ISD::SEXTLOAD, false))) {
+ SDValue ExtLoad =
+ DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
+ LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
+ AddToWorklist(ExtLoad.getNode());
+ return SDValue(N, 0); // Return N so it doesn't get rechecked!
+ }
}
// fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
- if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
- N0.hasOneUse() && ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
- ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) &&
- TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) {
+ if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
auto *LN0 = cast<LoadSDNode>(N0);
- SDValue ExtLoad =
- DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
- LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
- CombineTo(N, ExtLoad);
- CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
- return SDValue(N, 0); // Return N so it doesn't get rechecked!
+
+ if (N0.hasOneUse() && ExtVT == LN0->getMemoryVT() &&
+ ((!LegalOperations && LN0->isSimple()) &&
+ TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(),
+ LN0->getMemOperand()->getFlags(),
+ LN0->getAddressSpace(), ISD::SEXTLOAD, false))) {
+ SDValue ExtLoad =
+ DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
+ LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
+ return SDValue(N, 0); // Return N so it doesn't get rechecked!
+ }
}
// fold (sext_inreg (masked_load x)) -> (sext_masked_load x)
@@ -16268,7 +16335,9 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Frozen ? N0.getOperand(0) : N0)) {
if (ExtVT == Ld->getMemoryVT() && Ld->hasNUsesOfValue(1, 0) &&
Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD &&
- TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) {
+ TLI.isLoadLegal(VT, ExtVT, Ld->getAlign(),
+ Ld->getMemOperand()->getFlags(), Ld->getAddressSpace(),
+ ISD::SEXTLOAD, false)) {
SDValue ExtMaskedLoad = DAG.getMaskedLoad(
VT, DL, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
@@ -19726,20 +19795,22 @@ SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
}
// fold (fpext (load x)) -> (fpext (fptrunc (extload x)))
- if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
- TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) {
+ if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse()) {
LoadSDNode *LN0 = cast<LoadSDNode>(N0);
- SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT,
- LN0->getChain(),
- LN0->getBasePtr(), N0.getValueType(),
- LN0->getMemOperand());
- CombineTo(N, ExtLoad);
- CombineTo(
- N0.getNode(),
- DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), ExtLoad,
- DAG.getIntPtrConstant(1, SDLoc(N0), /*isTarget=*/true)),
- ExtLoad.getValue(1));
- return SDValue(N, 0); // Return N so it doesn't get rechecked!
+ if (TLI.isLoadLegalOrCustom(VT, N0.getValueType(), LN0->getAlign(),
+ LN0->getMemOperand()->getFlags(),
+ LN0->getAddressSpace(), ISD::EXTLOAD, false)) {
+ SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(),
+ LN0->getBasePtr(), N0.getValueType(),
+ LN0->getMemOperand());
+ CombineTo(N, ExtLoad);
+ CombineTo(
+ N0.getNode(),
+ DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), ExtLoad,
+ DAG.getIntPtrConstant(1, SDLoc(N0), /*isTarget=*/true)),
+ ExtLoad.getValue(1));
+ return SDValue(N, 0); // Return N so it doesn't get rechecked!
+ }
}
if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
@@ -22829,9 +22900,18 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
if (TLI.isTruncStoreLegal(LegalizedStoredValTy, StoreTy) &&
TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy,
DAG.getMachineFunction()) &&
- TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValTy, StoreTy) &&
- TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValTy, StoreTy) &&
- TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValTy, StoreTy) &&
+ TLI.isLoadLegal(
+ LegalizedStoredValTy, StoreTy, FirstLoad->getAlign(),
+ FirstLoad->getMemOperand()->getFlags(),
+ FirstLoad->getAddressSpace(), ISD::ZEXTLOAD, false) &&
+ TLI.isLoadLegal(
+ LegalizedStoredValTy, StoreTy, FirstLoad->getAlign(),
+ FirstLoad->getMemOperand()->getFlags(),
+ FirstLoad->getAddressSpace(), ISD::SEXTLOAD, false) &&
+ TLI.isLoadLegal(
+ LegalizedStoredValTy, StoreTy, FirstLoad->getAlign(),
+ FirstLoad->getMemOperand()->getFlags(),
+ FirstLoad->getAddressSpace(), ISD::EXTLOAD, false) &&
TLI.allowsMemoryAccess(Context, DL, StoreTy,
*FirstInChain->getMemOperand(), &IsFastSt) &&
IsFastSt &&
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 9ab0c9d475e94..9b08e9c338951 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -352,7 +352,12 @@ SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
// Only do this if the target has a native EXTLOAD instruction from
// smaller type.
- TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
+ TLI.isLoadLegal(
+ OrigVT, SVT, Align(DAG.getDataLayout().getPrefTypeAlign(SVT.getTypeForEVT(*DAG.getContext()))),
+ MachineMemOperand::Flags::MOLoad,
+ MachinePointerInfo::getConstantPool(DAG.getMachineFunction())
+ .getAddrSpace(),
+ ISD::EXTLOAD, false) &&
TLI.ShouldShrinkFPConstant(OrigVT)) {
Type *SType = SVT.getTypeForEVT(*DAG.getContext());
LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(
@@ -744,8 +749,9 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
// nice to have an effective generic way of getting these benefits...
// Until such a way is found, don't insist on promoting i1 here.
(SrcVT != MVT::i1 ||
- TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
- TargetLowering::Promote)) {
+ TLI.getLoadAction(Node->getValueType(0), MVT::i1, LD->getAlign(),
+ LD->getMemOperand()->getFlags(), LD->getAddressSpace(),
+ ExtType, false) == TargetLowering::Promote)) {
// Promote to a byte-sized load if not loading an integral number of
// bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
unsigned NewWidth = SrcVT.getStoreSizeInBits();
@@ -855,9 +861,11 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
Chain = Ch;
} else {
bool isCustom = false;
- switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
- SrcVT.getSimpleVT())) {
- default: llvm_unreachable("This action is not supported yet!");
+ switch (TLI.getLoadAction(Node->getValueType(0), SrcVT.getSimpleVT(),
+ LD->getAlign(), LD->getMemOperand()->getFlags(),
+ LD->getAddressSpace(), ExtType, false)) {
+ default:
+ llvm_unreachable("This action is not supported yet!");
case TargetLowering::Custom:
isCustom = true;
[[fallthrough]];
@@ -884,13 +892,17 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
case TargetLowering::Expand: {
EVT DestVT = Node->getValueType(0);
- if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
+ if (!TLI.isLoadLegal(DestVT, SrcVT, LD->getAlign(),
+ LD->getMemOperand()->getFlags(),
+ LD->getAddressSpace(), ISD::EXTLOAD, false)) {
// If the source type is not legal, see if there is a legal extload to
// an intermediate type that we can then extend further.
EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
(TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
- TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
+ TLI.isLoadLegal(LoadVT, SrcVT, LD->getAlign(),
+ LD->getMemOperand()->getFlags(),
+ LD->getAddressSpace(), ExtType, false))) {
// If we are loading a legal type, this is a non-extload followed by a
// full extend.
ISD::LoadExtType MidExtType =
@@ -1853,7 +1865,9 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
if ((SrcVT.bitsGT(SlotVT) &&
!TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
(SlotVT.bitsLT(DestVT) &&
- !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
+ !TLI.isLoadLegalOrCustom(
+ DestVT, SlotVT, DestAlign, MachineMemOperand::Flags::MOLoad,
+ DAG.getDataLayout().getAllocaAddrSpace(), ISD::EXTLOAD, false)))
return SDValue();
// Create the stack frame object.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 81184f709bd8c..a0c35c3579f16 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -298,7 +298,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
ISD::LoadExtType ExtType = LD->getExtensionType();
EVT LoadedVT = LD->getMemoryVT();
if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
- Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
+ Action = TLI.getLoadAction(LD->getValueType(0), LoadedVT, LD->getAlign(),
+ LD->getMemOperand()->getFlags(),
+ LD->getAddressSpace(), ExtType, false);
break;
}
case ISD::STORE: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index a10b93063dc27..449aa81445285 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12731,9 +12731,12 @@ SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
if (ResultVT.bitsGT(VecEltVT)) {
// If the result type of vextract is wider than the load, then issue an
// extending load instead.
- ISD::LoadExtType ExtType = isLoadExtLegal(ISD::ZEXTLOAD, ResultVT, VecEltVT)
- ? ISD::ZEXTLOAD
- : ISD::EXTLOAD;
+ ISD::LoadExtType ExtType =
+ isLoadLegal(ResultVT, VecEltVT, Alignment,
+ OriginalLoad->getMemOperand()->getFlags(),
+ OriginalLoad->getAddressSpace(), ISD::ZEXTLOAD, false)
+ ? ISD::ZEXTLOAD
+ : ISD::EXTLOAD;
Load = DAG.getExtLoad(ExtType, DL, ResultVT, OriginalLoad->getChain(),
NewPtr, MPI, VecEltVT, Alignment,
OriginalLoad->getMemOperand()->getFlags(),
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index c6707162d9b9e..abd77855a0ec0 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -20,6 +20,7 @@
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/Analysis.h"
+#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
@@ -2029,6 +2030,28 @@ MVT TargetLoweringBase::getPreferredSwitchConditionType(LLVMContext &Context,
return getRegisterType(Context, ConditionVT);
}
+TargetLoweringBase::LegalizeAction TargetLoweringBase::getLoadAction(
+ EVT ValVT, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags,
+ unsigned AddrSpace, unsigned ExtType, bool Atomic) const {
+ if (ValVT.isExtended() || MemVT.isExtended())
+ return Expand;
+ unsigned ValI = (unsigned)ValVT.getSimpleVT().SimpleTy;
+ unsigned MemI = (unsigned)MemVT.getSimpleVT().SimpleTy;
+ assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::VALUETYPE_SIZE &&
+ MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!");
+ unsigned Shift = 4 * ExtType;
+
+ if (Atomic) {
+ LegalizeAction Action =
+ (LegalizeAction)((AtomicLoadExtActions[ValI][MemI] >> Shift) & 0xf);
+ assert((Action == Legal || Action == Expand) &&
+ "Unsupported atomic load extension action.");
+ return Action;
+ }
+
+ return (LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf);
+}
+
/// Get the EVTs and ArgFlags collections that represent the legalized return
/// type of the given function. This does not require a DAG or a return value,
/// and is suitable for use before any DAGs for the function are constructed.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 842655d0ca0e9..47b1e9b355f0f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7045,7 +7045,9 @@ bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
// results in just one set of predicate unpacks at the start, instead of
// multiple sets of vector unpacks after each load.
if (auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal->getOperand(0))) {
- if (!isLoadExtLegalOrCustom(ISD::ZEXTLOAD, ExtVT, Ld->getValueType(0))) {
+ if (!isLoadLegalOrCustom(ExtVT, Ld->getValueType(0), Ld->getAlign(),
+ Ld->getMemOperand()->getFlags(),
+ Ld->getAddressSpace(), ISD::ZEXTLOAD, false)) {
// Disable extending masked loads for fixed-width for now, since the code
// quality doesn't look great.
if (!ExtVT.isScalableVector())
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3ec2bf9b19360..6b3d9cdf72239 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -61640,7 +61640,9 @@ static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG,
? ISD::SEXTLOAD
: ISD::ZEXTLOAD;
EVT MemVT = VT.changeVectorElementType(*DAG.getContext(), SVT);
- if (TLI.isLoadExtLegal(Ext, VT, MemVT)) {
+ if (TLI.isLoadLegal(VT, MemVT, Ld->getAlign(),
+ Ld->getMemOperand()->getFlags(),
+ Ld->getAddressSpace(), Ext, false)) {
SDValue Load = DAG.getExtLoad(
Ext, DL, VT, Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(),
MemVT, Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());
More information about the llvm-commits
mailing list