[llvm] 9fe5700 - [AArch64] Add support for v8.4a `ldapur`/`stlur`
Antonio Frighetto via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 30 11:30:34 PDT 2023
Author: Antonio Frighetto
Date: 2023-10-30T19:27:48+01:00
New Revision: 9fe5700611de180c2b5cfc0422eaebe1d027a826
URL: https://github.com/llvm/llvm-project/commit/9fe5700611de180c2b5cfc0422eaebe1d027a826
DIFF: https://github.com/llvm/llvm-project/commit/9fe5700611de180c2b5cfc0422eaebe1d027a826.diff
LOG: [AArch64] Add support for v8.4a `ldapur`/`stlur`
AArch64 backend now features v8.4a atomic Load-Acquire
RCpc and Store-Release register unscaled support.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
llvm/lib/Target/AArch64/AArch64InstrAtomics.td
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-rcpc_immo.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 38759a2474518fc..7617dccdeee397f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -997,6 +997,15 @@ static bool isWorthFoldingADDlow(SDValue N) {
return true;
}
+/// Check if the immediate offset is valid as a scaled immediate.
+static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range,
+ unsigned Size) {
+ if ((Offset & (Size - 1)) == 0 && Offset >= 0 &&
+ Offset < (Range << Log2_32(Size)))
+ return true;
+ return false;
+}
+
/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
/// immediate" address. The "Size" argument is the size in bytes of the memory
/// reference, which determines the scale.
@@ -1092,7 +1101,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
int64_t RHSC = (int64_t)RHS->getZExtValue();
unsigned Scale = Log2_32(Size);
- if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
+ if (isValidAsScaledImmediate(RHSC, 0x1000, Size)) {
Base = N.getOperand(0);
if (Base.getOpcode() == ISD::FrameIndex) {
int FI = cast<FrameIndexSDNode>(Base)->getIndex();
@@ -1130,10 +1139,6 @@ bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
return false;
if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
int64_t RHSC = RHS->getSExtValue();
- // If the offset is valid as a scaled immediate, don't match here.
- if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 &&
- RHSC < (0x1000 << Log2_32(Size)))
- return false;
if (RHSC >= -256 && RHSC < 256) {
Base = N.getOperand(0);
if (Base.getOpcode() == ISD::FrameIndex) {
@@ -1312,11 +1317,10 @@ bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
// LDR X2, [BaseReg, X0]
if (isa<ConstantSDNode>(RHS)) {
int64_t ImmOff = (int64_t)cast<ConstantSDNode>(RHS)->getZExtValue();
- unsigned Scale = Log2_32(Size);
// Skip the immediate can be selected by load/store addressing mode.
// Also skip the immediate can be encoded by a single ADD (SUB is also
// checked by using -ImmOff).
- if ((ImmOff % Size == 0 && ImmOff >= 0 && ImmOff < (0x1000 << Scale)) ||
+ if (isValidAsScaledImmediate(ImmOff, 0x1000, Size) ||
isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
return false;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index fa5a8515ed92eca..0002db52b1995c0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -573,3 +573,34 @@ let Predicates = [HasRCPC3, HasNEON] in {
(i64 (bitconvert (v1f64 VecListOne64:$Vt)))),
(STL1 (SUBREG_TO_REG (i64 0), VecListOne64:$Vt, dsub), (i64 0), GPR64sp:$Rn)>;
}
+
+// v8.4a FEAT_LRCPC2 patterns
+let Predicates = [HasRCPC_IMMO] in {
+ // Load-Acquire RCpc Register unscaled loads
+ def : Pat<(acquiring_load<atomic_load_az_8>
+ (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
+ (LDAPURBi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(acquiring_load<atomic_load_az_16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
+ (LDAPURHi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(acquiring_load<atomic_load_32>
+ (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
+ (LDAPURi GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(acquiring_load<atomic_load_64>
+ (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
+ (LDAPURXi GPR64sp:$Rn, simm9:$offset)>;
+
+ // Store-Release Register unscaled stores
+ def : Pat<(releasing_store<atomic_store_8>
+ (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
+ (STLURBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
+ (STLURHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_32>
+ (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
+ (STLURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_64>
+ (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
+ (STLURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
+}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index ee42612c0fcdd2a..069a283dd311e50 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -94,7 +94,7 @@ def HasTLB_RMI : Predicate<"Subtarget->hasTLB_RMI()">,
def HasFlagM : Predicate<"Subtarget->hasFlagM()">,
AssemblerPredicateWithAll<(all_of FeatureFlagM), "flagm">;
-def HasRCPC_IMMO : Predicate<"Subtarget->hasRCPCImm()">,
+def HasRCPC_IMMO : Predicate<"Subtarget->hasRCPC_IMMO()">,
AssemblerPredicateWithAll<(all_of FeatureRCPC_IMMO), "rcpc-immo">;
def HasFPARMv8 : Predicate<"Subtarget->hasFPARMv8()">,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 2089bfba5ff37c6..88516967515a58b 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -7397,9 +7397,6 @@ AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
return std::nullopt;
RHSC = RHSOp1.getCImm()->getSExtValue();
- // If the offset is valid as a scaled immediate, don't match here.
- if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
- return std::nullopt;
if (RHSC >= -256 && RHSC < 256) {
MachineOperand &Base = RootDef->getOperand(1);
return {{
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
index 05f37a4e440eb03..cea15419e67c859 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
@@ -36,8 +36,7 @@ define i8 @load_atomic_i8_aligned_monotonic_const(ptr readonly %ptr) {
define i8 @load_atomic_i8_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_acquire:
-; CHECK: add x8, x0, #4
-; CHECK: ldaprb w0, [x8]
+; CHECK: ldapurb w0, [x0, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
%r = load atomic i8, ptr %gep acquire, align 1
ret i8 %r
@@ -45,8 +44,7 @@ define i8 @load_atomic_i8_aligned_acquire(ptr %ptr) {
define i8 @load_atomic_i8_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_acquire_const:
-; CHECK: add x8, x0, #4
-; CHECK: ldaprb w0, [x8]
+; CHECK: ldapurb w0, [x0, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
%r = load atomic i8, ptr %gep acquire, align 1
ret i8 %r
@@ -104,8 +102,7 @@ define i16 @load_atomic_i16_aligned_monotonic_const(ptr readonly %ptr) {
define i16 @load_atomic_i16_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_acquire:
-; CHECK: add x8, x0, #8
-; CHECK: ldaprh w0, [x8]
+; CHECK: ldapurh w0, [x0, #8]
%gep = getelementptr inbounds i16, ptr %ptr, i32 4
%r = load atomic i16, ptr %gep acquire, align 2
ret i16 %r
@@ -113,8 +110,7 @@ define i16 @load_atomic_i16_aligned_acquire(ptr %ptr) {
define i16 @load_atomic_i16_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_acquire_const:
-; CHECK: add x8, x0, #8
-; CHECK: ldaprh w0, [x8]
+; CHECK: ldapurh w0, [x0, #8]
%gep = getelementptr inbounds i16, ptr %ptr, i32 4
%r = load atomic i16, ptr %gep acquire, align 2
ret i16 %r
@@ -172,8 +168,7 @@ define i32 @load_atomic_i32_aligned_monotonic_const(ptr readonly %ptr) {
define i32 @load_atomic_i32_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_acquire:
-; CHECK: add x8, x0, #16
-; CHECK: ldapr w0, [x8]
+; CHECK: ldapur w0, [x0, #16]
%gep = getelementptr inbounds i32, ptr %ptr, i32 4
%r = load atomic i32, ptr %gep acquire, align 4
ret i32 %r
@@ -181,8 +176,7 @@ define i32 @load_atomic_i32_aligned_acquire(ptr %ptr) {
define i32 @load_atomic_i32_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_acquire_const:
-; CHECK: add x8, x0, #16
-; CHECK: ldapr w0, [x8]
+; CHECK: ldapur w0, [x0, #16]
%gep = getelementptr inbounds i32, ptr %ptr, i32 4
%r = load atomic i32, ptr %gep acquire, align 4
ret i32 %r
@@ -240,8 +234,7 @@ define i64 @load_atomic_i64_aligned_monotonic_const(ptr readonly %ptr) {
define i64 @load_atomic_i64_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_acquire:
-; CHECK: add x8, x0, #32
-; CHECK: ldapr x0, [x8]
+; CHECK: ldapur x0, [x0, #32]
%gep = getelementptr inbounds i64, ptr %ptr, i32 4
%r = load atomic i64, ptr %gep acquire, align 8
ret i64 %r
@@ -249,8 +242,7 @@ define i64 @load_atomic_i64_aligned_acquire(ptr %ptr) {
define i64 @load_atomic_i64_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_acquire_const:
-; CHECK: add x8, x0, #32
-; CHECK: ldapr x0, [x8]
+; CHECK: ldapur x0, [x0, #32]
%gep = getelementptr inbounds i64, ptr %ptr, i32 4
%r = load atomic i64, ptr %gep acquire, align 8
ret i64 %r
@@ -376,8 +368,7 @@ define i8 @load_atomic_i8_unaligned_monotonic_const(ptr readonly %ptr) {
define i8 @load_atomic_i8_unaligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_acquire:
-; CHECK: add x8, x0, #4
-; CHECK: ldaprb w0, [x8]
+; CHECK: ldapurb w0, [x0, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
%r = load atomic i8, ptr %gep acquire, align 1
ret i8 %r
@@ -385,8 +376,7 @@ define i8 @load_atomic_i8_unaligned_acquire(ptr %ptr) {
define i8 @load_atomic_i8_unaligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_acquire_const:
-; CHECK: add x8, x0, #4
-; CHECK: ldaprb w0, [x8]
+; CHECK: ldapurb w0, [x0, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
%r = load atomic i8, ptr %gep acquire, align 1
ret i8 %r
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-rcpc_immo.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-rcpc_immo.ll
index 86cb738c5799ddb..4f461571c55824f 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-rcpc_immo.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-rcpc_immo.ll
@@ -20,8 +20,7 @@ define void @store_atomic_i8_aligned_monotonic(i8 %value, ptr %ptr) {
define void @store_atomic_i8_aligned_release(i8 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i8_aligned_release:
-; CHECK: add x8, x1, #4
-; CHECK: stlrb w0, [x8]
+; CHECK: stlurb w0, [x1, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
store atomic i8 %value, ptr %gep release, align 1
ret void
@@ -29,8 +28,7 @@ define void @store_atomic_i8_aligned_release(i8 %value, ptr %ptr) {
define void @store_atomic_i8_aligned_seq_cst(i8 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i8_aligned_seq_cst:
-; CHECK: add x8, x1, #4
-; CHECK: stlrb w0, [x8]
+; CHECK: stlurb w0, [x1, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
store atomic i8 %value, ptr %gep seq_cst, align 1
ret void
@@ -54,8 +52,7 @@ define void @store_atomic_i16_aligned_monotonic(i16 %value, ptr %ptr) {
define void @store_atomic_i16_aligned_release(i16 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i16_aligned_release:
-; CHECK: add x8, x1, #8
-; CHECK: stlrh w0, [x8]
+; CHECK: stlurh w0, [x1, #8]
%gep = getelementptr inbounds i16, ptr %ptr, i32 4
store atomic i16 %value, ptr %gep release, align 2
ret void
@@ -63,8 +60,7 @@ define void @store_atomic_i16_aligned_release(i16 %value, ptr %ptr) {
define void @store_atomic_i16_aligned_seq_cst(i16 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i16_aligned_seq_cst:
-; CHECK: add x8, x1, #8
-; CHECK: stlrh w0, [x8]
+; CHECK: stlurh w0, [x1, #8]
%gep = getelementptr inbounds i16, ptr %ptr, i32 4
store atomic i16 %value, ptr %gep seq_cst, align 2
ret void
@@ -88,8 +84,7 @@ define void @store_atomic_i32_aligned_monotonic(i32 %value, ptr %ptr) {
define void @store_atomic_i32_aligned_release(i32 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i32_aligned_release:
-; CHECK: add x8, x1, #16
-; CHECK: stlr w0, [x8]
+; CHECK: stlur w0, [x1, #16]
%gep = getelementptr inbounds i32, ptr %ptr, i32 4
store atomic i32 %value, ptr %gep release, align 4
ret void
@@ -97,8 +92,7 @@ define void @store_atomic_i32_aligned_release(i32 %value, ptr %ptr) {
define void @store_atomic_i32_aligned_seq_cst(i32 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i32_aligned_seq_cst:
-; CHECK: add x8, x1, #16
-; CHECK: stlr w0, [x8]
+; CHECK: stlur w0, [x1, #16]
%gep = getelementptr inbounds i32, ptr %ptr, i32 4
store atomic i32 %value, ptr %gep seq_cst, align 4
ret void
@@ -122,8 +116,7 @@ define void @store_atomic_i64_aligned_monotonic(i64 %value, ptr %ptr) {
define void @store_atomic_i64_aligned_release(i64 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i64_aligned_release:
-; CHECK: add x8, x1, #32
-; CHECK: stlr x0, [x8]
+; CHECK: stlur x0, [x1, #32]
%gep = getelementptr inbounds i64, ptr %ptr, i32 4
store atomic i64 %value, ptr %gep release, align 8
ret void
@@ -131,8 +124,7 @@ define void @store_atomic_i64_aligned_release(i64 %value, ptr %ptr) {
define void @store_atomic_i64_aligned_seq_cst(i64 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i64_aligned_seq_cst:
-; CHECK: add x8, x1, #32
-; CHECK: stlr x0, [x8]
+; CHECK: stlur x0, [x1, #32]
%gep = getelementptr inbounds i64, ptr %ptr, i32 4
store atomic i64 %value, ptr %gep seq_cst, align 8
ret void
@@ -191,8 +183,7 @@ define void @store_atomic_i8_unaligned_monotonic(i8 %value, ptr %ptr) {
define void @store_atomic_i8_unaligned_release(i8 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i8_unaligned_release:
-; CHECK: add x8, x1, #4
-; CHECK: stlrb w0, [x8]
+; CHECK: stlurb w0, [x1, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
store atomic i8 %value, ptr %gep release, align 1
ret void
@@ -200,8 +191,7 @@ define void @store_atomic_i8_unaligned_release(i8 %value, ptr %ptr) {
define void @store_atomic_i8_unaligned_seq_cst(i8 %value, ptr %ptr) {
; CHECK-LABEL: store_atomic_i8_unaligned_seq_cst:
-; CHECK: add x8, x1, #4
-; CHECK: stlrb w0, [x8]
+; CHECK: stlurb w0, [x1, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
store atomic i8 %value, ptr %gep seq_cst, align 1
ret void
More information about the llvm-commits
mailing list