[llvm] [LSV] Don't vectorize load chains across ordered atomics (PR #208631)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 15:59:06 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Justin Lebar (jlebar)
<details>
<summary>Changes</summary>
When the LoadStoreVectorizer finds a chain of loads to merge into a single vectorized load, it hoists all of the constituent loads up to the location of the first load in the chain.
This is obviously not safe if there are any may-alias stores in the middle of the chain. But it's *also* not safe if there are `acquire` *loads* in the middle of the chain.
There's a similar problem with stores. A chain of stores is vectorized by merging them all into the final store in the chain. This is not safe if there is an intervening `release` store.
---
Patch is 20.76 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/208631.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp (+4-2)
- (modified) llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll (+353-53)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 897285f722089..29496eeccce39 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1298,8 +1298,10 @@ bool Vectorizer::isSafeToMove(
if (!I->mayReadOrWriteMemory())
continue;
- // Loads can be reordered with other loads.
- if (IsLoadChain && isa<LoadInst>(I))
+ // Loads can be reordered with other unordered loads. Ordered atomics
+ // act as reordering barriers, via getModRefInfo below.
+ if (auto *LI = dyn_cast<LoadInst>(I);
+ IsLoadChain && LI && LI->isUnordered())
continue;
// Stores can be sunk below invariant loads.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
index 782dba25f063d..6c7511c8db37f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Check that the load/store vectorizer is willing to move loads/stores across
@@ -15,11 +16,15 @@ declare void @fn_writeonly() #3
declare void @fn_readonly() #4
declare void @fn_readnone() #5
-; CHECK-LABEL: @load_fn
-; CHECK: load
-; CHECK: call void @fn()
-; CHECK: load
define void @load_fn(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -28,11 +33,15 @@ define void @load_fn(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind
-; CHECK: load
-; CHECK: call void @fn_nounwind()
-; CHECK: load
define void @load_fn_nounwind(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_nounwind() #[[ATTR0]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -41,11 +50,15 @@ define void @load_fn_nounwind(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind_writeonly
-; CHECK: load
-; CHECK: call void @fn_nounwind_writeonly()
-; CHECK: load
define void @load_fn_nounwind_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_nounwind_writeonly() #[[ATTR1:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -54,10 +67,15 @@ define void @load_fn_nounwind_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind_readonly
-; CHECK-DAG: load <2 x i32>
-; CHECK-DAG: call void @fn_nounwind_readonly()
define void @load_fn_nounwind_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: call void @fn_nounwind_readonly() #[[ATTR2:[0-9]+]]
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -66,11 +84,15 @@ define void @load_fn_nounwind_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_readonly
-; CHECK: load
-; CHECK: call void @fn_readonly
-; CHECK: load
define void @load_fn_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_readonly() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -79,11 +101,15 @@ define void @load_fn_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_writeonly
-; CHECK: load
-; CHECK: call void @fn_writeonly()
-; CHECK: load
define void @load_fn_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_writeonly() #[[ATTR3:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -92,10 +118,15 @@ define void @load_fn_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_readnone
-; CHECK-DAG: load <2 x i32>
-; CHECK-DAG: call void @fn_readnone()
define void @load_fn_readnone(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_readnone(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: call void @fn_readnone() #[[ATTR5:[0-9]+]]
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -108,11 +139,15 @@ define void @load_fn_readnone(ptr %p) #0 {
; Same tests, but now for stores instead of loads.
; ------------------------------------------------
-; CHECK-LABEL: @store_fn
-; CHECK: store
-; CHECK: call void @fn()
-; CHECK: store
define void @store_fn(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -121,11 +156,15 @@ define void @store_fn(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind
-; CHECK: store
-; CHECK: call void @fn_nounwind()
-; CHECK: store
define void @store_fn_nounwind(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind() #[[ATTR0]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -134,11 +173,15 @@ define void @store_fn_nounwind(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind_writeonly
-; CHECK: store
-; CHECK: call void @fn_nounwind_writeonly()
-; CHECK: store
define void @store_fn_nounwind_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind_writeonly() #[[ATTR1]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -147,11 +190,15 @@ define void @store_fn_nounwind_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind_readonly
-; CHECK: store
-; CHECK: call void @fn_nounwind_readonly()
-; CHECK: store
define void @store_fn_nounwind_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind_readonly() #[[ATTR2]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -160,11 +207,15 @@ define void @store_fn_nounwind_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_readonly
-; CHECK: store
-; CHECK: call void @fn_readonly
-; CHECK: store
define void @store_fn_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_readonly() #[[ATTR4]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -173,11 +224,15 @@ define void @store_fn_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_writeonly
-; CHECK: store
-; CHECK: call void @fn_writeonly()
-; CHECK: store
define void @store_fn_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_writeonly() #[[ATTR3]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -187,10 +242,13 @@ define void @store_fn_writeonly(ptr %p) #0 {
}
; This is the only store idiom we can vectorize.
-; CHECK-LABEL: @store_fn_readnone
-; CHECK-DAG: store <2 x i32>
-; CHECK-DAG: call void @fn_readnone()
define void @store_fn_readnone(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_readnone(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @fn_readnone() #[[ATTR5]]
+; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p, align 8
@@ -207,3 +265,245 @@ attributes #3 = { writeonly }
attributes #4 = { readonly }
; readnone implies nounwind, so no need to test separately
attributes #5 = { nounwind willreturn readnone }
+
+; --------------------------------------------------------
+; Load chains across atomics.
+; --------------------------------------------------------
+; Vectorizing a load chain hoists its later loads up to the first one.
+; Hoisting a load above an intervening acquire operation would break
+; release/acquire synchronization, so ordered atomics conservatively act as
+; reordering barriers; only unordered loads may be crossed.
+
+; Hoisting the load of %p.1 above the acquire load would break the
+; release/acquire message-passing idiom.
+
+define <2 x i32> @load_acquire(ptr %p, ptr %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_acquire(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] acquire, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag acquire, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Monotonic imposes no ordering on other locations, but LSV is conservative.
+
+define <2 x i32> @load_monotonic(ptr %p, ptr %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_monotonic(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] monotonic, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag monotonic, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Unordered loads can be crossed.
+
+define <2 x i32> @load_unordered(ptr %p, ptr %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_unordered(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] unordered, align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V01]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag unordered, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A volatile load already terminates the region LSV vectorizes within.
+
+define <2 x i32> @load_volatile(ptr %p, ptr %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_volatile(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load volatile i32, ptr [[FLAG]], align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load volatile i32, ptr %flag, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A release store permits hoisting loads in principle; LSV is conservative.
+
+define <2 x i32> @load_release_store(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_release_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] release, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ store atomic i32 1, ptr %flag release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Same for RMWs...
+
+define <2 x i32> @load_release_rmw(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_release_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 release, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; ...and cmpxchg.
+
+define <2 x i32> @load_release_cmpxchg(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_release_cmpxchg(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = cmpxchg ptr [[FLAG]], i32 0, i32 1 release monotonic, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = cmpxchg ptr %flag, i32 0, i32 1 release monotonic, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; --------------------------------------------------------
+; Store chains across atomics.
+; --------------------------------------------------------
+; The mirror image: vectorizing a store chain sinks its earlier stores down
+; to the last one, so again ordered atomics act as barriers.
+
+; An acquire load permits sinking stores in principle; LSV is conservative.
+
+define void @store_acquire_load(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define void @store_acquire_load(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] acquire, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %f = load atomic i32, ptr %flag acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; Same for RMWs.
+
+define void @store_acquire_rmw(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define void @store_acquire_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 acquire, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %old = atomicrmw add ptr %flag...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/208631
More information about the llvm-commits
mailing list