[llvm] [AArch64] Use LD1/ST1 in strict-align mode (PR #211513)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 23 03:10:09 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Oliver Stannard (ostannard)
<details>
<summary>Changes</summary>
When compiling with +strict-align, we currently scalarise all fixed
width vector loads and stores where the alignment is less than the
vector size, because the LDR/STR instructions require alignment matching
the vector size. However, the LD1/ST1 instructions only require the
alignment to match the size of the individual elements, so we can use
them in more cases.
This patch:
* Adds a check in allowsMisalignedMemoryAccesses, to prevent memory
accesses being scalarised when not needed.
* Doubles the tablegen patterns for the vector LDR/STR instructions, to
restrict them to vector-size aligned pointers in +strict-align mode,
and leave them unrestricted otherwise.
---
Patch is 499.59 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/211513.diff
6 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp (+10-3)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+23-3)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+1-1)
- (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.td (+398-108)
- (added) llvm/test/CodeGen/AArch64/vector-ldst-align.ll (+8227)
- (modified) llvm/test/CodeGen/AArch64/vector-ldst-offset.ll (+3448-2)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 0862c5754da60..4535274a2db0c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -1837,6 +1837,7 @@ bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
ISD::LoadExtType ExtType = LD->getExtensionType();
bool InsertTo64 = false;
+ bool OffsetIsXZR = false;
if (VT == MVT::i64)
Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
else if (VT == MVT::i32) {
@@ -1884,9 +1885,13 @@ bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
} else if (VT == MVT::f32) {
Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
} else if (VT == MVT::f64 ||
- (VT.is64BitVector() && Subtarget->isLittleEndian())) {
+ (VT.is64BitVector() && Subtarget->isLittleEndian() &&
+ (!Subtarget->requiresStrictAlign() ||
+ LD->getAlign() >= VT.getStoreSize()))) {
Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
- } else if (VT.is128BitVector() && Subtarget->isLittleEndian()) {
+ } else if (VT.is128BitVector() && Subtarget->isLittleEndian() &&
+ (!Subtarget->requiresStrictAlign() ||
+ LD->getAlign() >= VT.getStoreSize())) {
Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
} else if (VT.is64BitVector()) {
if (IsPre || OffsetVal != 8)
@@ -1907,6 +1912,7 @@ bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
default:
llvm_unreachable("Expected vector element to be a power of 2");
}
+ OffsetIsXZR = true;
} else if (VT.is128BitVector()) {
if (IsPre || OffsetVal != 16)
return false;
@@ -1926,13 +1932,14 @@ bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
default:
llvm_unreachable("Expected vector element to be a power of 2");
}
+ OffsetIsXZR = true;
} else
return false;
SDValue Chain = LD->getChain();
SDValue Base = LD->getBasePtr();
SDLoc dl(N);
// LD1 encodes an immediate offset by using XZR as the offset register.
- SDValue Offset = (VT.isVector() && !Subtarget->isLittleEndian())
+ SDValue Offset = OffsetIsXZR
? CurDAG->getRegister(AArch64::XZR, MVT::i64)
: CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
SDValue Ops[] = { Base, Offset, Chain };
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 213a0042461ae..ed2aad99c0945 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2435,6 +2435,10 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
// When little-endian we can use ordinary d and q register loads/stores for
// vector types, but when big-endian we need to use structure load/store which
// only allow post-index addressing.
+ // With +strict-align, we also need to use LD1/ST1 when the alignment is
+ // less than the vector size, but we can still use LDR/STR for more-aligned
+ // accesses, so these are marked as legal here, and the invalid cases are
+ // rejected in getIndexedAddressParts.
if (Subtarget->isLittleEndian()) {
for (unsigned im = (unsigned)ISD::PRE_INC;
im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
@@ -3153,6 +3157,15 @@ bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
return true;
}
+ // For NEON, we can use LD1/ST1 when the alignment is less than the size of
+ // the vector, but greater than or equal to the size of the elements.
+ if (Subtarget->requiresStrictAlign() && VT.isFixedLengthVector() &&
+ (VT.getSizeInBits() == 64 || VT.getSizeInBits() == 128)) {
+ unsigned ElementSizeBits = VT.getScalarSizeInBits();
+ if (ElementSizeBits % 8 == 0 && Alignment >= Align(ElementSizeBits / 8))
+ return true;
+ }
+
if (Subtarget->requiresStrictAlign())
return false;
@@ -31558,7 +31571,8 @@ bool AArch64TargetLowering::isIndexingLegal(MachineInstr &MI, Register Base,
bool AArch64TargetLowering::getIndexedAddressParts(SDNode *N, SDNode *Op,
SDValue &Base,
SDValue &Offset,
- SelectionDAG &DAG) const {
+ SelectionDAG &DAG,
+ ISD::MemIndexedMode AM) const {
if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
return false;
@@ -31602,6 +31616,12 @@ bool AArch64TargetLowering::getIndexedAddressParts(SDNode *N, SDNode *Op,
if (!Subtarget->isLittleEndian() && MemType.isVector() &&
(uint64_t)RHSC != MemType.getStoreSize())
return false;
+ // Likewise, when compiling with +strict-align we use LD1/ST1 when the
+ // alignment is less than the vector size.
+ if (Subtarget->requiresStrictAlign() && MemType.isVector() &&
+ cast<MemSDNode>(N)->getAlign() < MemType.getStoreSize() &&
+ ((uint64_t)RHSC != MemType.getStoreSize() || AM != ISD::POST_INC))
+ return false;
// Always emit pre-inc/post-inc addressing mode. Use negated constant offset
// when dealing with subtraction.
Offset = DAG.getConstant(RHSC, SDLoc(N), RHS->getValueType(0));
@@ -31634,7 +31654,7 @@ bool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
if (IsVolatile)
return false;
- if (!getIndexedAddressParts(N, Ptr.getNode(), Base, Offset, DAG))
+ if (!getIndexedAddressParts(N, Ptr.getNode(), Base, Offset, DAG, ISD::PRE_INC))
return false;
AM = ISD::PRE_INC;
return true;
@@ -31663,7 +31683,7 @@ bool AArch64TargetLowering::getPostIndexedAddressParts(
if (IsVolatile)
return false;
- if (!getIndexedAddressParts(N, Op, Base, Offset, DAG))
+ if (!getIndexedAddressParts(N, Op, Base, Offset, DAG, ISD::POST_INC))
return false;
// Post-indexing updates the base, so it's not a valid transform
// if that's not the same as the load's pointer.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 9c2ea0faee5ec..840c75948cdb3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -903,7 +903,7 @@ class AArch64TargetLowering : public TargetLowering {
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
bool getIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
- SDValue &Offset, SelectionDAG &DAG) const;
+ SDValue &Offset, SelectionDAG &DAG, ISD::MemIndexedMode AM) const;
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
ISD::MemIndexedMode &AM,
SelectionDAG &DAG) const override;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index a660a3f11bdb2..bff10762c5639 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -436,6 +436,10 @@ def AArch64LocalRecover : SDNode<"ISD::LOCAL_RECOVER",
def AllowMisalignedMemAccesses
: Predicate<"!Subtarget->requiresStrictAlign()">;
+def DisallowMisalignedMemAccesses
+ : Predicate<"Subtarget->requiresStrictAlign()">;
+def IsBEOrDisallowMisalignedMemAccesses
+ : Predicate<"!Subtarget->isLittleEndian() || Subtarget->requiresStrictAlign()">;
def UseWzrToVecMove : Predicate<"Subtarget->useWzrToVecMove()">;
@@ -762,6 +766,35 @@ def topbitsallzero64: PatLeaf<(i64 GPR64:$src), [{
return VT && VT->maskedValueIsZero(Reg, APInt::getHighBitsSet(64, 63)); }];
}
+// Loads and stores with a minimum alignment.
+class load_aligned<int align_bytes> : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> {
+ let IsLoad = true;
+ let IsNonExtLoad = true;
+ let MinAlignment = align_bytes;
+}
+class store_aligned<int align_bytes> : PatFrag<(ops node:$val, node:$ptr),
+ (unindexedstore node:$val, node:$ptr)> {
+ let IsStore = true;
+ let IsTruncStore = false;
+ let MinAlignment = align_bytes;
+}
+class pre_store_aligned<int align_bytes> : PatFrag<(ops node:$val, node:$base, node:$offset),
+ (istore node:$val, node:$base, node:$offset), [{
+ ISD::MemIndexedMode AM = cast<StoreSDNode>(N)->getAddressingMode();
+ Align A = cast<StoreSDNode>(N)->getAlign();
+ return (AM == ISD::PRE_INC || AM == ISD::PRE_DEC) && A >= }] # align_bytes # [{;
+}]> {
+ let MinAlignment = align_bytes;
+}
+class post_store_aligned<int align_bytes> : PatFrag<(ops node:$val, node:$ptr, node:$offset),
+ (istore node:$val, node:$ptr, node:$offset), [{
+ ISD::MemIndexedMode AM = cast<StoreSDNode>(N)->getAddressingMode();
+ Align A = cast<StoreSDNode>(N)->getAlign();
+ return (AM == ISD::POST_INC || AM == ISD::POST_DEC) && A >= }] # align_bytes # [{;
+}]> {
+ let MinAlignment = align_bytes;
+}
+
// Node definitions.
// Compare-and-branch
def AArch64CB : SDNode<"AArch64ISD::CB", SDT_AArch64cb, [SDNPHasChain]>;
@@ -3915,43 +3948,48 @@ defm LDRSW : Load32RO<0b10, 0, 0b10, GPR64, "ldrsw", i64, sextloadi32>;
// Pre-fetch.
defm PRFM : PrefetchRO<0b11, 0, 0b10, "prfm">;
-// Match all load 64 bits width whose type is compatible with FPR64
multiclass VecROLoadPat<ROAddrMode ro, ValueType VecTy,
- Instruction LOADW, Instruction LOADX> {
-
- def : Pat<(VecTy (load (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend))),
- (LOADW GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
+ Instruction LOADW, Instruction LOADX,
+ int vector_align, list<Predicate> preds> {
+ let Predicates = !listconcat([AllowMisalignedMemAccesses], preds) in {
+ def : Pat<(VecTy (load (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend))),
+ (LOADW GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
+
+ def : Pat<(VecTy (load (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend))),
+ (LOADX GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
+ }
+ let Predicates = !listconcat([DisallowMisalignedMemAccesses], preds) in {
+ def : Pat<(VecTy (load_aligned<vector_align> (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend))),
+ (LOADW GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
- def : Pat<(VecTy (load (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend))),
- (LOADX GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
+ def : Pat<(VecTy (load_aligned<vector_align> (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend))),
+ (LOADX GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
+ }
}
-let AddedComplexity = 10 in {
-let Predicates = [IsLE] in {
- // We must do vector loads with LD1 in big-endian.
- defm : VecROLoadPat<ro64, v2i32, LDRDroW, LDRDroX>;
- defm : VecROLoadPat<ro64, v2f32, LDRDroW, LDRDroX>;
- defm : VecROLoadPat<ro64, v8i8, LDRDroW, LDRDroX>;
- defm : VecROLoadPat<ro64, v4i16, LDRDroW, LDRDroX>;
- defm : VecROLoadPat<ro64, v4f16, LDRDroW, LDRDroX>;
- defm : VecROLoadPat<ro64, v4bf16, LDRDroW, LDRDroX>;
-}
+// We must do vector loads with LD1 in big-endian, or when the alignment is
+// less than the size of the vector.
-defm : VecROLoadPat<ro64, v1i64, LDRDroW, LDRDroX>;
-defm : VecROLoadPat<ro64, v1f64, LDRDroW, LDRDroX>;
+// Match all load 64 bits width whose type is compatible with FPR64
+let AddedComplexity = 10 in {
+defm : VecROLoadPat<ro64, v2i32, LDRDroW, LDRDroX, 8, [IsLE]>;
+defm : VecROLoadPat<ro64, v2f32, LDRDroW, LDRDroX, 8, [IsLE]>;
+defm : VecROLoadPat<ro64, v8i8, LDRDroW, LDRDroX, 8, [IsLE]>;
+defm : VecROLoadPat<ro64, v4i16, LDRDroW, LDRDroX, 8, [IsLE]>;
+defm : VecROLoadPat<ro64, v4f16, LDRDroW, LDRDroX, 8, [IsLE]>;
+defm : VecROLoadPat<ro64, v4bf16, LDRDroW, LDRDroX, 8, [IsLE]>;
+defm : VecROLoadPat<ro64, v1i64, LDRDroW, LDRDroX, 8, []>;
+defm : VecROLoadPat<ro64, v1f64, LDRDroW, LDRDroX, 8, []>;
// Match all load 128 bits width whose type is compatible with FPR128
-let Predicates = [IsLE] in {
- // We must do vector loads with LD1 in big-endian.
- defm : VecROLoadPat<ro128, v2i64, LDRQroW, LDRQroX>;
- defm : VecROLoadPat<ro128, v2f64, LDRQroW, LDRQroX>;
- defm : VecROLoadPat<ro128, v4i32, LDRQroW, LDRQroX>;
- defm : VecROLoadPat<ro128, v4f32, LDRQroW, LDRQroX>;
- defm : VecROLoadPat<ro128, v8i16, LDRQroW, LDRQroX>;
- defm : VecROLoadPat<ro128, v8f16, LDRQroW, LDRQroX>;
- defm : VecROLoadPat<ro128, v8bf16, LDRQroW, LDRQroX>;
- defm : VecROLoadPat<ro128, v16i8, LDRQroW, LDRQroX>;
-}
+defm : VecROLoadPat<ro128, v2i64, LDRQroW, LDRQroX, 16, [IsLE]>;
+defm : VecROLoadPat<ro128, v2f64, LDRQroW, LDRQroX, 16, [IsLE]>;
+defm : VecROLoadPat<ro128, v4i32, LDRQroW, LDRQroX, 16, [IsLE]>;
+defm : VecROLoadPat<ro128, v4f32, LDRQroW, LDRQroX, 16, [IsLE]>;
+defm : VecROLoadPat<ro128, v8i16, LDRQroW, LDRQroX, 16, [IsLE]>;
+defm : VecROLoadPat<ro128, v8f16, LDRQroW, LDRQroX, 16, [IsLE]>;
+defm : VecROLoadPat<ro128, v8bf16, LDRQroW, LDRQroX, 16, [IsLE]>;
+defm : VecROLoadPat<ro128, v16i8, LDRQroW, LDRQroX, 16, [IsLE]>;
} // AddedComplexity = 10
// zextload -> i64
@@ -4034,9 +4072,11 @@ defm LDRQ : LoadUI<0b00, 1, 0b11, FPR128Op, uimm12s16, "ldr",
def : Pat <(bf16 (load (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))),
(LDRHui GPR64sp:$Rn, uimm12s2:$offset)>;
+// We must use LD1 to perform vector loads in big-endian, or when the
+// alignment is less than the vector size.
+
// Match all load 64 bits width whose type is compatible with FPR64
-let Predicates = [IsLE] in {
- // We must use LD1 to perform vector loads in big-endian.
+let Predicates = [IsLE, AllowMisalignedMemAccesses] in {
def : Pat<(v2f32 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
(LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
def : Pat<(v8i8 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
@@ -4055,9 +4095,23 @@ def : Pat<(v1f64 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
def : Pat<(v1i64 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
(LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
+let Predicates = [IsLE, DisallowMisalignedMemAccesses] in {
+ def : Pat<(v2f32 (load_aligned<8> (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
+ (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
+ def : Pat<(v8i8 (load_aligned<8> (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
+ (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
+ def : Pat<(v4i16 (load_aligned<8> (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
+ (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
+ def : Pat<(v2i32 (load_aligned<8> (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
+ (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
+ def : Pat<(v4f16 (load_aligned<8> (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
+ (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
+ def : Pat<(v4bf16 (load_aligned<8> (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
+ (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
+}
+
// Match all load 128 bits width whose type is compatible with FPR128
-let Predicates = [IsLE] in {
- // We must use LD1 to perform vector loads in big-endian.
+let Predicates = [IsLE, AllowMisalignedMemAccesses] in {
def : Pat<(v4f32 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
(LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
def : Pat<(v2f64 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
@@ -4078,6 +4132,25 @@ let Predicates = [IsLE] in {
def : Pat<(f128 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
(LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+let Predicates = [IsLE, DisallowMisalignedMemAccesses] in {
+ def : Pat<(v4f32 (load_aligned<16> (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
+ (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+ def : Pat<(v2f64 (load_aligned<16> (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
+ (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+ def : Pat<(v16i8 (load_aligned<16> (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
+ (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+ def : Pat<(v8i16 (load_aligned<16> (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
+ (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+ def : Pat<(v4i32 (load_aligned<16> (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
+ (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+ def : Pat<(v2i64 (load_aligned<16> (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
+ (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+ def : Pat<(v8f16 (load_aligned<16> (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
+ (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+ def : Pat<(v8bf16 (load_aligned<16> (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
+ (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
+}
+
defm LDRHH : LoadUI<0b01, 0, 0b01, GPR32, uimm12s2, "ldrh",
[(set GPR32:$Rt,
(zextloadi16 (am_indexed16 GPR64sp:$Rn,
@@ -4232,7 +4305,7 @@ def : Pat <(bf16 (load (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
(LDURHi GPR64sp:$Rn, simm9:$offset)>;
// Match all load 64 bits width whose type is compatible with FPR64
-let Predicates = [IsLE] in {
+let Predicates = [IsLE, AllowMisalignedMemAccesses] in {
def : Pat<(v2f32 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
(LDURDi GPR64sp:$Rn, simm9:$offset)>;
def : Pat<(v2i32 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
@@ -4249,8 +4322,21 @@ def : Pat<(v1f64 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
def : Pat<(v1i64 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
(LDURDi GPR64sp:$Rn, simm9:$offset)>;
+let Predicates = [IsLE, DisallowMisalignedMemAccesses] in {
+ def : Pat<(v2f32 (load_aligned<8> (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
+ (LDURDi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v2i32 (load_aligned<8> (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
+ (LDURDi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v4i16 (load_aligned<8> (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
+ (LDURDi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v8i8 (load_aligned<8> (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
+ (LDURDi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v4f16 (load_aligned<8> (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
+ (LDURDi GPR64sp:$Rn, simm9:$offset)>;
+}
+
// Match all load 128 bits width whose type is compatible with FPR128
-let Predicates = [IsLE] in {
+let Predicates = [IsLE, AllowMisalignedMemAccesses] in {
def : Pat<(v2f64 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
(LDURQi GPR64sp:$Rn, simm9:$offset)>;
def : Pat<(v2i64 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
@@ -4267,6 +4353,23 @@ let Predicates = [IsLE] in {
(LDURQi GPR64sp:$Rn, simm9:$offset)>;
}
+let Predicates = [IsLE, DisallowMisalignedMemAccesses] in {
+ def : Pat<(v2f64 (load_aligned<16> (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
+ (LDURQi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v2i64 (load_aligned<16> (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
+ (LDURQi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v4f32 (load_aligned<16> (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
+ (LDURQi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v4i32 (load_aligned<16> (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
+ (LDURQi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v8i16 (load_aligned<16> (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
+ (LDURQi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v16i8 (load_aligned<16> (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
+ (LDURQi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(v8f16 (load_aligned<16> (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
+ (LDURQi GPR64sp:$Rn, simm9:$offset)>;
+}
+
// anyext -> zext
def : Pat<(i32 (extloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
(LDURHHi GPR64sp:$Rn, simm9:$offset)>;
@@ -4772,44 +4875,54 @@ let AddedComplexity = 10 in {
defm : TruncStoreFrom64ROPat<ro32, truncstorei32, STRWroW, STRWroX>;
}
+
multiclass VecROSto...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/211513
More information about the llvm-commits
mailing list