[llvm] 153b1e3 - [AArch64] Add patterns for relaxed atomic ld/st into fp registers
Danila Malyutin via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 25 04:34:22 PST 2022
Author: Danila Malyutin
Date: 2022-01-25T15:33:37+03:00
New Revision: 153b1e3cba1e0469bfa6c72208d91708c219e6a6
URL: https://github.com/llvm/llvm-project/commit/153b1e3cba1e0469bfa6c72208d91708c219e6a6
DIFF: https://github.com/llvm/llvm-project/commit/153b1e3cba1e0469bfa6c72208d91708c219e6a6.diff
LOG: [AArch64] Add patterns for relaxed atomic ld/st into fp registers
Adds patterns to match integer loads/stores bitcasted to fp values
Fixes https://github.com/llvm/llvm-project/issues/52927
Differential Revision: https://reviews.llvm.org/D117573
Added:
llvm/test/CodeGen/AArch64/relaxed-fp-atomics.ll
Modified:
llvm/lib/Target/AArch64/AArch64InstrAtomics.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 84573dac7e41..b220929514f9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -102,6 +102,34 @@ def : Pat<(relaxed_load<atomic_load_64>
(am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
(LDURXi GPR64sp:$Rn, simm9:$offset)>;
+// FP 32-bit loads
+def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
+ ro_Wextend32:$extend))))),
+ (LDRSroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
+def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
+ ro_Xextend32:$extend))))),
+ (LDRSroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
+def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
+ uimm12s8:$offset))))),
+ (LDRSui GPR64sp:$Rn, uimm12s8:$offset)>;
+def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32>
+ (am_unscaled32 GPR64sp:$Rn, simm9:$offset))))),
+ (LDURSi GPR64sp:$Rn, simm9:$offset)>;
+
+// FP 64-bit loads
+def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
+ ro_Wextend64:$extend))))),
+ (LDRDroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
+def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
+ ro_Xextend64:$extend))))),
+ (LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
+def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
+ uimm12s8:$offset))))),
+ (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
+def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64>
+ (am_unscaled64 GPR64sp:$Rn, simm9:$offset))))),
+ (LDURDi GPR64sp:$Rn, simm9:$offset)>;
+
//===----------------------------------
// Atomic stores
//===----------------------------------
@@ -196,6 +224,38 @@ def : Pat<(relaxed_store<atomic_store_64>
(am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
(STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
+// FP 32-bit stores
+def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
+ ro_Wextend32:$extend),
+ (i32 (bitconvert (f32 FPR32Op:$val)))),
+ (STRSroW FPR32Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
+def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
+ ro_Xextend32:$extend),
+ (i32 (bitconvert (f32 FPR32Op:$val)))),
+ (STRSroX FPR32Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
+def : Pat<(relaxed_store<atomic_store_32>
+ (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))),
+ (STRSui FPR32Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
+def : Pat<(relaxed_store<atomic_store_32>
+ (am_unscaled32 GPR64sp:$Rn, simm9:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))),
+ (STURSi FPR32Op:$val, GPR64sp:$Rn, simm9:$offset)>;
+
+// FP 64-bit stores
+def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
+ ro_Wextend64:$extend),
+ (i64 (bitconvert (f64 FPR64Op:$val)))),
+ (STRDroW FPR64Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
+def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
+ ro_Xextend64:$extend),
+ (i64 (bitconvert (f64 FPR64Op:$val)))),
+ (STRDroX FPR64Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
+def : Pat<(relaxed_store<atomic_store_64>
+ (am_indexed64 GPR64sp:$Rn, uimm12s4:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),
+ (STRDui FPR64Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
+def : Pat<(relaxed_store<atomic_store_64>
+ (am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),
+ (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>;
+
//===----------------------------------
// Low-level exclusive operations
//===----------------------------------
diff --git a/llvm/test/CodeGen/AArch64/relaxed-fp-atomics.ll b/llvm/test/CodeGen/AArch64/relaxed-fp-atomics.ll
new file mode 100644
index 000000000000..1f8ba6da2464
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/relaxed-fp-atomics.ll
@@ -0,0 +1,94 @@
+; PR52927: Relaxed atomics can load to/store from fp regs directly
+; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone | FileCheck %s
+
+define float @atomic_load_relaxed_f32(float* %p, i32 %off32, i64 %off64) #0 {
+; CHECK-LABEL: atomic_load_relaxed_f32:
+ %ptr_unsigned = getelementptr float, float* %p, i32 4095
+ %val_unsigned = load atomic float, float* %ptr_unsigned monotonic, align 4
+; CHECK: ldr {{s[0-9]+}}, [x0, #16380]
+
+ %ptr_regoff = getelementptr float, float* %p, i32 %off32
+ %val_regoff = load atomic float, float* %ptr_regoff unordered, align 4
+ %tot1 = fadd float %val_unsigned, %val_regoff
+; CHECK: ldr {{s[0-9]+}}, [x0, w1, sxtw #2]
+
+ %ptr_regoff64 = getelementptr float, float* %p, i64 %off64
+ %val_regoff64 = load atomic float, float* %ptr_regoff64 monotonic, align 4
+ %tot2 = fadd float %tot1, %val_regoff64
+; CHECK: ldr {{s[0-9]+}}, [x0, x2, lsl #2]
+
+ %ptr_unscaled = getelementptr float, float* %p, i32 -64
+ %val_unscaled = load atomic float, float* %ptr_unscaled unordered, align 4
+ %tot3 = fadd float %tot2, %val_unscaled
+; CHECK: ldur {{s[0-9]+}}, [x0, #-256]
+
+ ret float %tot3
+}
+
+define double @atomic_load_relaxed_f64(double* %p, i32 %off32, i64 %off64) #0 {
+; CHECK-LABEL: atomic_load_relaxed_f64:
+ %ptr_unsigned = getelementptr double, double* %p, i32 4095
+ %val_unsigned = load atomic double, double* %ptr_unsigned monotonic, align 8
+; CHECK: ldr {{d[0-9]+}}, [x0, #32760]
+
+ %ptr_regoff = getelementptr double, double* %p, i32 %off32
+ %val_regoff = load atomic double, double* %ptr_regoff unordered, align 8
+ %tot1 = fadd double %val_unsigned, %val_regoff
+; CHECK: ldr {{d[0-9]+}}, [x0, w1, sxtw #3]
+
+ %ptr_regoff64 = getelementptr double, double* %p, i64 %off64
+ %val_regoff64 = load atomic double, double* %ptr_regoff64 monotonic, align 8
+ %tot2 = fadd double %tot1, %val_regoff64
+; CHECK: ldr {{d[0-9]+}}, [x0, x2, lsl #3]
+
+ %ptr_unscaled = getelementptr double, double* %p, i32 -32
+ %val_unscaled = load atomic double, double* %ptr_unscaled unordered, align 8
+ %tot3 = fadd double %tot2, %val_unscaled
+; CHECK: ldur {{d[0-9]+}}, [x0, #-256]
+
+ ret double %tot3
+}
+
+define void @atomic_store_relaxed_f32(float* %p, i32 %off32, i64 %off64, float %val) #0 {
+; CHECK-LABEL: atomic_store_relaxed_f32:
+ %ptr_unsigned = getelementptr float, float* %p, i32 4095
+ store atomic float %val, float* %ptr_unsigned monotonic, align 4
+; CHECK: str {{s[0-9]+}}, [x0, #16380]
+
+ %ptr_regoff = getelementptr float, float* %p, i32 %off32
+ store atomic float %val, float* %ptr_regoff unordered, align 4
+; CHECK: str {{s[0-9]+}}, [x0, w1, sxtw #2]
+
+ %ptr_regoff64 = getelementptr float, float* %p, i64 %off64
+ store atomic float %val, float* %ptr_regoff64 monotonic, align 4
+; CHECK: str {{s[0-9]+}}, [x0, x2, lsl #2]
+
+ %ptr_unscaled = getelementptr float, float* %p, i32 -64
+ store atomic float %val, float* %ptr_unscaled unordered, align 4
+; CHECK: stur {{s[0-9]+}}, [x0, #-256]
+
+ ret void
+}
+
+define void @atomic_store_relaxed_f64(double* %p, i32 %off32, i64 %off64, double %val) #0 {
+; CHECK-LABEL: atomic_store_relaxed_f64:
+ %ptr_unsigned = getelementptr double, double* %p, i32 4095
+ store atomic double %val, double* %ptr_unsigned monotonic, align 8
+; CHECK: str {{d[0-9]+}}, [x0, #32760]
+
+ %ptr_regoff = getelementptr double, double* %p, i32 %off32
+ store atomic double %val, double* %ptr_regoff unordered, align 8
+; CHECK: str {{d[0-9]+}}, [x0, w1, sxtw #3]
+
+ %ptr_regoff64 = getelementptr double, double* %p, i64 %off64
+ store atomic double %val, double* %ptr_regoff64 unordered, align 8
+; CHECK: str {{d[0-9]+}}, [x0, x2, lsl #3]
+
+ %ptr_unscaled = getelementptr double, double* %p, i32 -32
+ store atomic double %val, double* %ptr_unscaled monotonic, align 8
+; CHECK: stur {{d[0-9]+}}, [x0, #-256]
+
+ ret void
+}
+
+attributes #0 = { nounwind }
More information about the llvm-commits
mailing list