[llvm] [X86] Move atomic load of fp -> int promotion into the DAG (PR #118793)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 5 10:25:10 PST 2024
https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/118793
>From 9b0a33b3f4e9d7da6e755e36722e72396fb9f064 Mon Sep 17 00:00:00 2001
From: jofrn <jofernau at amd.com>
Date: Mon, 25 Nov 2024 15:05:53 -0500
Subject: [PATCH 1/6] [Verifier] Allow vector type in atomic load and store
Vector types on atomics are assumed to be invalid by the verifier. However,
this type can be valid if it is lowered by codegen.
---
llvm/docs/LangRef.rst | 8 ++++----
llvm/lib/IR/Verifier.cpp | 14 ++++++++------
llvm/test/Verifier/atomics.ll | 15 ++++++++-------
3 files changed, 20 insertions(+), 17 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 79bdd25c18f1fd..32ba5ebdec6d37 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -10956,8 +10956,8 @@ If the ``load`` is marked as ``atomic``, it takes an extra :ref:`ordering
<ordering>` and optional ``syncscope("<target-scope>")`` argument. The
``release`` and ``acq_rel`` orderings are not valid on ``load`` instructions.
Atomic loads produce :ref:`defined <memmodel>` results when they may see
-multiple atomic stores. The type of the pointee must be an integer, pointer, or
-floating-point type whose bit width is a power of two greater than or equal to
+multiple atomic stores. The type of the pointee must be an integer, pointer,
+floating-point, or vector type whose bit width is a power of two greater than or equal to
eight and less than or equal to a target-specific size limit. ``align`` must be
explicitly specified on atomic loads. Note: if the alignment is not greater or
equal to the size of the `<value>` type, the atomic operation is likely to
@@ -11097,8 +11097,8 @@ If the ``store`` is marked as ``atomic``, it takes an extra :ref:`ordering
<ordering>` and optional ``syncscope("<target-scope>")`` argument. The
``acquire`` and ``acq_rel`` orderings aren't valid on ``store`` instructions.
Atomic loads produce :ref:`defined <memmodel>` results when they may see
-multiple atomic stores. The type of the pointee must be an integer, pointer, or
-floating-point type whose bit width is a power of two greater than or equal to
+multiple atomic stores. The type of the pointee must be an integer, pointer,
+floating-point, or vector type whose bit width is a power of two greater than or equal to
eight and less than or equal to a target-specific size limit. ``align`` must be
explicitly specified on atomic stores. Note: if the alignment is not greater or
equal to the size of the `<value>` type, the atomic operation is likely to
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 55de486e90e190..6f847e3b3fc70c 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4255,9 +4255,10 @@ void Verifier::visitLoadInst(LoadInst &LI) {
Check(LI.getOrdering() != AtomicOrdering::Release &&
LI.getOrdering() != AtomicOrdering::AcquireRelease,
"Load cannot have Release ordering", &LI);
- Check(ElTy->isIntOrPtrTy() || ElTy->isFloatingPointTy(),
- "atomic load operand must have integer, pointer, or floating point "
- "type!",
+ Check(ElTy->getScalarType()->isIntOrPtrTy() ||
+ ElTy->getScalarType()->isFloatingPointTy(),
+ "atomic load operand must have integer, pointer, floating point, "
+ "or vector type!",
ElTy, &LI);
checkAtomicMemAccessSize(ElTy, &LI);
} else {
@@ -4281,9 +4282,10 @@ void Verifier::visitStoreInst(StoreInst &SI) {
Check(SI.getOrdering() != AtomicOrdering::Acquire &&
SI.getOrdering() != AtomicOrdering::AcquireRelease,
"Store cannot have Acquire ordering", &SI);
- Check(ElTy->isIntOrPtrTy() || ElTy->isFloatingPointTy(),
- "atomic store operand must have integer, pointer, or floating point "
- "type!",
+ Check(ElTy->getScalarType()->isIntOrPtrTy() ||
+ ElTy->getScalarType()->isFloatingPointTy(),
+ "atomic store operand must have integer, pointer, floating point, "
+ "or vector type!",
ElTy, &SI);
checkAtomicMemAccessSize(ElTy, &SI);
} else {
diff --git a/llvm/test/Verifier/atomics.ll b/llvm/test/Verifier/atomics.ll
index f835b98b243456..17bf5a0528d738 100644
--- a/llvm/test/Verifier/atomics.ll
+++ b/llvm/test/Verifier/atomics.ll
@@ -1,14 +1,15 @@
; RUN: not opt -passes=verify < %s 2>&1 | FileCheck %s
+; CHECK: atomic store operand must have integer, pointer, floating point, or vector type!
+; CHECK: atomic load operand must have integer, pointer, floating point, or vector type!
-; CHECK: atomic store operand must have integer, pointer, or floating point type!
-; CHECK: atomic load operand must have integer, pointer, or floating point type!
+%ty = type { i32 };
-define void @foo(ptr %P, <1 x i64> %v) {
- store atomic <1 x i64> %v, ptr %P unordered, align 8
+define void @foo(ptr %P, %ty %v) {
+ store atomic %ty %v, ptr %P unordered, align 8
ret void
}
-define <1 x i64> @bar(ptr %P) {
- %v = load atomic <1 x i64>, ptr %P unordered, align 8
- ret <1 x i64> %v
+define %ty @bar(ptr %P) {
+ %v = load atomic %ty, ptr %P unordered, align 8
+ ret %ty %v
}
>From aba2a030efe870593a3fc3d8ccf6eace7c70522c Mon Sep 17 00:00:00 2001
From: jofrn <jofernau at amd.com>
Date: Wed, 4 Dec 2024 13:30:12 -0500
Subject: [PATCH 2/6] Update Assembler/atomic test
---
llvm/test/Assembler/atomic.ll | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index a44dcccc16bef1..f1027d5d3fbde4 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -52,6 +52,15 @@ define void @f(ptr %x) {
; CHECK: atomicrmw volatile usub_sat ptr %x, i32 10 syncscope("agent") monotonic
atomicrmw volatile usub_sat ptr %x, i32 10 syncscope("agent") monotonic
+ ; CHECK : load atomic <1 x i32>, ptr %x unordered, align 4
+ load atomic <1 x i32>, ptr %x unordered, align 4
+ ; CHECK : store atomic <1 x i32> splat (i32 3), ptr %x release, align 4
+ store atomic <1 x i32> <i32 3>, ptr %x release, align 4
+ ; CHECK : load atomic <2 x i32>, ptr %x unordered, align 4
+ load atomic <2 x i32>, ptr %x unordered, align 4
+ ; CHECK : store atomic <2 x i32> <i32 3, i32 4>, ptr %x release, align 4
+ store atomic <2 x i32> <i32 3, i32 4>, ptr %x release, align 4
+
; CHECK: fence syncscope("singlethread") release
fence syncscope("singlethread") release
; CHECK: fence seq_cst
>From 10b57a1b1ee586ac492657fd07713ee5b75e1908 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 7 Oct 2024 13:47:50 -0400
Subject: [PATCH 3/6] [SelectionDAG] Legalize vector types for atomic load
Scalarize vector of atomic load in SelectionDAG.
---
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 16 ++++++++++++++++
.../CodeGen/Generic/atomic-scalarization.ll | 17 +++++++++++++++++
3 files changed, 34 insertions(+)
create mode 100644 llvm/test/CodeGen/Generic/atomic-scalarization.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 1703149aca7463..0086405825cd5c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -860,6 +860,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_ExpOp(SDNode *N);
SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N);
SDValue ScalarizeVecRes_LOAD(LoadSDNode *N);
+ SDValue ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N);
SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
SDValue ScalarizeVecRes_VSELECT(SDNode *N);
SDValue ScalarizeVecRes_SELECT(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 465128099f4447..eb3841f48c0e5a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -60,6 +60,9 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
case ISD::FPOWI: R = ScalarizeVecRes_ExpOp(N); break;
case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
+ case ISD::ATOMIC_LOAD:
+ R = ScalarizeVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N));
+ break;
case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
@@ -451,6 +454,19 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
return Op;
}
+SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
+
+ SDValue Result = DAG.getAtomic(ISD::ATOMIC_LOAD, SDLoc(N),
+ N->getMemoryVT().getVectorElementType(),
+ N->getValueType(0).getVectorElementType(),
+ N->getChain(), N->getBasePtr(), N->getMemOperand());
+
+ // Legalize the chain result - switch anything that used the old chain to
+ // use the new one.
+ ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
+ return Result;
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
assert(N->isUnindexed() && "Indexed vector load?");
diff --git a/llvm/test/CodeGen/Generic/atomic-scalarization.ll b/llvm/test/CodeGen/Generic/atomic-scalarization.ll
new file mode 100644
index 00000000000000..3f611323172884
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/atomic-scalarization.ll
@@ -0,0 +1,17 @@
+; RUN: llc %s --print-after-isel --disable-verify 2>&1 | FileCheck %s
+
+define i32 @atomic_scalar() {
+; CHECK: # After Instruction Selection:
+; CHECK-NEXT: # Machine code for function atomic_scalar: IsSSA, TracksLiveness
+; CHECK-NEXT: Frame Objects:
+; CHECK-NEXT: fi#0: size=4, align=4, at location [SP+8]
+; CHECK: bb.0 (%ir-block.0):
+; CHECK-NEXT: %0:gr32 = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (dereferenceable load acquire (s32) from %ir.1)
+; CHECK-NEXT: $eax = COPY %0:gr32
+; CHECK-NEXT: RET 0, $eax
+; CHECK: # End machine code for function atomic_scalar.
+ %1 = alloca <1 x i32>
+ %2 = load atomic <1 x i32>, ptr %1 acquire, align 4
+ %3 = extractelement <1 x i32> %2, i32 0
+ ret i32 %3
+}
>From a95ff19bde0aea2ca084fbe8398c9099c45d4b1b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 7 Oct 2024 14:18:52 -0400
Subject: [PATCH 4/6] Moved test and checking mir after last X86 pass
---
.../CodeGen/Generic/atomic-scalarization.ll | 17 -----------------
llvm/test/CodeGen/X86/atomic-scalarization.ll | 16 ++++++++++++++++
2 files changed, 16 insertions(+), 17 deletions(-)
delete mode 100644 llvm/test/CodeGen/Generic/atomic-scalarization.ll
create mode 100644 llvm/test/CodeGen/X86/atomic-scalarization.ll
diff --git a/llvm/test/CodeGen/Generic/atomic-scalarization.ll b/llvm/test/CodeGen/Generic/atomic-scalarization.ll
deleted file mode 100644
index 3f611323172884..00000000000000
--- a/llvm/test/CodeGen/Generic/atomic-scalarization.ll
+++ /dev/null
@@ -1,17 +0,0 @@
-; RUN: llc %s --print-after-isel --disable-verify 2>&1 | FileCheck %s
-
-define i32 @atomic_scalar() {
-; CHECK: # After Instruction Selection:
-; CHECK-NEXT: # Machine code for function atomic_scalar: IsSSA, TracksLiveness
-; CHECK-NEXT: Frame Objects:
-; CHECK-NEXT: fi#0: size=4, align=4, at location [SP+8]
-; CHECK: bb.0 (%ir-block.0):
-; CHECK-NEXT: %0:gr32 = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (dereferenceable load acquire (s32) from %ir.1)
-; CHECK-NEXT: $eax = COPY %0:gr32
-; CHECK-NEXT: RET 0, $eax
-; CHECK: # End machine code for function atomic_scalar.
- %1 = alloca <1 x i32>
- %2 = load atomic <1 x i32>, ptr %1 acquire, align 4
- %3 = extractelement <1 x i32> %2, i32 0
- ret i32 %3
-}
diff --git a/llvm/test/CodeGen/X86/atomic-scalarization.ll b/llvm/test/CodeGen/X86/atomic-scalarization.ll
new file mode 100644
index 00000000000000..22d240bd855180
--- /dev/null
+++ b/llvm/test/CodeGen/X86/atomic-scalarization.ll
@@ -0,0 +1,16 @@
+; RUN: llc %s --mtriple=x86_64 -print-after=unpack-mi-bundles -disable-verify 2>&1 | FileCheck %s
+
+define i32 @atomic_scalar() {
+; CHECK: # *** IR Dump After Unpack machine instruction bundles (unpack-mi-bundles) ***:
+; CHECK-NEXT: # Machine code for function atomic_scalar: NoPHIs, TracksLiveness, NoVRegs, TiedOpsRewritten, TracksDebugUserValues
+; CHECK-NEXT: Frame Objects:
+; CHECK-NEXT: fi#0: size=4, align=4, at location [SP-4]
+; CHECK: bb.0 (%ir-block.0):
+; CHECK-NEXT: renamable $eax = MOV32rm $rsp, 1, $noreg, -4, $noreg :: (dereferenceable load acquire (s32) from %ir.1)
+; CHECK-NEXT: RET64 $eax
+; CHECK: # End machine code for function atomic_scalar.
+ %1 = alloca <1 x i32>
+ %2 = load atomic <1 x i32>, ptr %1 acquire, align 4
+ %3 = extractelement <1 x i32> %2, i32 0
+ ret i32 %3
+}
>From d9d4a1c3a89d6171be481cc6f03db5e7de56bc28 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 7 Oct 2024 14:44:00 -0400
Subject: [PATCH 5/6] Autogenerate test.
---
.../SelectionDAG/LegalizeVectorTypes.cpp | 8 ++---
llvm/test/CodeGen/X86/atomic-scalarization.ll | 33 +++++++++++--------
2 files changed, 23 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index eb3841f48c0e5a..bdd71b251f3941 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -456,10 +456,10 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
- SDValue Result = DAG.getAtomic(ISD::ATOMIC_LOAD, SDLoc(N),
- N->getMemoryVT().getVectorElementType(),
- N->getValueType(0).getVectorElementType(),
- N->getChain(), N->getBasePtr(), N->getMemOperand());
+ SDValue Result = DAG.getAtomic(
+ ISD::ATOMIC_LOAD, SDLoc(N), N->getMemoryVT().getVectorElementType(),
+ N->getValueType(0).getVectorElementType(), N->getChain(), N->getBasePtr(),
+ N->getMemOperand());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
diff --git a/llvm/test/CodeGen/X86/atomic-scalarization.ll b/llvm/test/CodeGen/X86/atomic-scalarization.ll
index 22d240bd855180..6560924ed99546 100644
--- a/llvm/test/CodeGen/X86/atomic-scalarization.ll
+++ b/llvm/test/CodeGen/X86/atomic-scalarization.ll
@@ -1,16 +1,21 @@
-; RUN: llc %s --mtriple=x86_64 -print-after=unpack-mi-bundles -disable-verify 2>&1 | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc %s --mtriple=x86_64 -o - | FileCheck %s
-define i32 @atomic_scalar() {
-; CHECK: # *** IR Dump After Unpack machine instruction bundles (unpack-mi-bundles) ***:
-; CHECK-NEXT: # Machine code for function atomic_scalar: NoPHIs, TracksLiveness, NoVRegs, TiedOpsRewritten, TracksDebugUserValues
-; CHECK-NEXT: Frame Objects:
-; CHECK-NEXT: fi#0: size=4, align=4, at location [SP-4]
-; CHECK: bb.0 (%ir-block.0):
-; CHECK-NEXT: renamable $eax = MOV32rm $rsp, 1, $noreg, -4, $noreg :: (dereferenceable load acquire (s32) from %ir.1)
-; CHECK-NEXT: RET64 $eax
-; CHECK: # End machine code for function atomic_scalar.
- %1 = alloca <1 x i32>
- %2 = load atomic <1 x i32>, ptr %1 acquire, align 4
- %3 = extractelement <1 x i32> %2, i32 0
- ret i32 %3
+define <1 x i32> @atomic_scalar_i32(ptr %x) {
+; CHECK-LABEL: atomic_scalar_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: retq
+ %ret = load atomic <1 x i32>, ptr %x acquire, align 4
+ ret <1 x i32> %ret
+}
+
+define <1 x bfloat> @atomic_scalar_bfloat(ptr %x) {
+; CHECK-LABEL: atomic_scalar_bfloat:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movzwl (%rdi), %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-NEXT: retq
+ %ret = load atomic <1 x bfloat>, ptr %x acquire, align 4
+ ret <1 x bfloat> %ret
}
>From da3e8f65d68c1fc03785c4014c850ee960dae9a5 Mon Sep 17 00:00:00 2001
From: jofrn <jofernau at amd.com>
Date: Thu, 5 Dec 2024 05:09:48 -0500
Subject: [PATCH 6/6] [SelectionDAG][X86] Add floating point promotion.
When lowering atomic vector types with floats, selection can fail since
this pattern is unsupported. To support this, floats can be casted to
an integer type of the same size.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 4 ++++
llvm/test/CodeGen/X86/atomic-scalarization.ll | 19 +++++++++++++++++++
2 files changed, 23 insertions(+)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9048d1d83f1874..7f4021f4beedcc 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2589,6 +2589,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(Op, MVT::f32, Promote);
}
+ setOperationPromotedToType(ISD::ATOMIC_LOAD, MVT::f16, MVT::i16);
+ setOperationPromotedToType(ISD::ATOMIC_LOAD, MVT::f32, MVT::i32);
+ setOperationPromotedToType(ISD::ATOMIC_LOAD, MVT::f64, MVT::i64);
+
// We have target-specific dag combine patterns for the following nodes:
setTargetDAGCombine({ISD::VECTOR_SHUFFLE,
ISD::SCALAR_TO_VECTOR,
diff --git a/llvm/test/CodeGen/X86/atomic-scalarization.ll b/llvm/test/CodeGen/X86/atomic-scalarization.ll
index 6560924ed99546..dfffb894a87ddb 100644
--- a/llvm/test/CodeGen/X86/atomic-scalarization.ll
+++ b/llvm/test/CodeGen/X86/atomic-scalarization.ll
@@ -10,6 +10,25 @@ define <1 x i32> @atomic_scalar_i32(ptr %x) {
ret <1 x i32> %ret
}
+define <1 x float> @atomic_scalar_float(ptr %x) {
+; CHECK-LABEL: atomic_scalar_float:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: retq
+ %ret = load atomic <1 x float>, ptr %x acquire, align 4
+ ret <1 x float> %ret
+}
+
+define <1 x half> @atomic_scalar_half(ptr %x) {
+; CHECK-LABEL: atomic_scalar_half:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movzwl (%rdi), %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-NEXT: retq
+ %ret = load atomic <1 x half>, ptr %x acquire, align 4
+ ret <1 x half> %ret
+}
+
define <1 x bfloat> @atomic_scalar_bfloat(ptr %x) {
; CHECK-LABEL: atomic_scalar_bfloat:
; CHECK: # %bb.0:
More information about the llvm-commits
mailing list