[llvm] [LSV] Don't vectorize load chains across ordered atomics (PR #208631)
Justin Lebar via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 15:53:58 PDT 2026
https://github.com/jlebar updated https://github.com/llvm/llvm-project/pull/208631
>From 34f1d651650c04e657617bdc712f9e2cefb65eef Mon Sep 17 00:00:00 2001
From: Justin Lebar <justin.lebar at gmail.com>
Date: Sat, 18 Jul 2026 20:35:55 +0000
Subject: [PATCH] [LSV] Don't vectorize load chains across ordered atomics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The LoadStoreVectorizer emits a merged load at the chain's first
element, hoisting every later element up to that point. isSafeToMove
skipped all loads in the hoisted-over region on the grounds that loads
reorder freely with loads — true only of unordered loads. An acquire
load is a one-way barrier: no later access may be hoisted above it,
including loads of unrelated locations.
define i32 @consume(ptr %data, ptr %flag) {
%lo = load i32, ptr %data, align 8
%f = load atomic i32, ptr %flag acquire, align 4
%p1 = getelementptr inbounds i8, ptr %data, i64 4
%hi = load i32, ptr %p1, align 4
...
}
$ opt -passes=load-store-vectorizer -mtriple=x86_64
%1 = load <2 x i32>, ptr %data, align 8 ; both words above the acquire
%f = load atomic i32, ptr %flag acquire, align 4
This breaks message passing: a producer stores data[1] = 2 and then
release-stores flag = 1; a consumer that observes %f == 1 is guaranteed
%hi == 2, but the rewritten code reads data[1] without waiting for the
acquire, so %hi can be stale.
Only skip unordered loads. Every ordered atomic then reaches
getModRefInfo and conservatively splits the chain. (getModRefInfo's one
gap there -- it ignored a cmpxchg's failure ordering -- is fixed
separately in "[AA] Use the cmpxchg merged ordering in getModRefInfo".)
---
.../Vectorize/LoadStoreVectorizer.cpp | 6 +-
.../NVPTX/merge-across-side-effects.ll | 406 +++++++++++++++---
2 files changed, 357 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 897285f722089..29496eeccce39 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1298,8 +1298,10 @@ bool Vectorizer::isSafeToMove(
if (!I->mayReadOrWriteMemory())
continue;
- // Loads can be reordered with other loads.
- if (IsLoadChain && isa<LoadInst>(I))
+ // Loads can be reordered with other unordered loads. Ordered atomics
+ // act as reordering barriers, via getModRefInfo below.
+ if (auto *LI = dyn_cast<LoadInst>(I);
+ IsLoadChain && LI && LI->isUnordered())
continue;
// Stores can be sunk below invariant loads.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
index 782dba25f063d..6c7511c8db37f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Check that the load/store vectorizer is willing to move loads/stores across
@@ -15,11 +16,15 @@ declare void @fn_writeonly() #3
declare void @fn_readonly() #4
declare void @fn_readnone() #5
-; CHECK-LABEL: @load_fn
-; CHECK: load
-; CHECK: call void @fn()
-; CHECK: load
define void @load_fn(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -28,11 +33,15 @@ define void @load_fn(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind
-; CHECK: load
-; CHECK: call void @fn_nounwind()
-; CHECK: load
define void @load_fn_nounwind(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_nounwind() #[[ATTR0]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -41,11 +50,15 @@ define void @load_fn_nounwind(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind_writeonly
-; CHECK: load
-; CHECK: call void @fn_nounwind_writeonly()
-; CHECK: load
define void @load_fn_nounwind_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_nounwind_writeonly() #[[ATTR1:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -54,10 +67,15 @@ define void @load_fn_nounwind_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind_readonly
-; CHECK-DAG: load <2 x i32>
-; CHECK-DAG: call void @fn_nounwind_readonly()
define void @load_fn_nounwind_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: call void @fn_nounwind_readonly() #[[ATTR2:[0-9]+]]
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -66,11 +84,15 @@ define void @load_fn_nounwind_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_readonly
-; CHECK: load
-; CHECK: call void @fn_readonly
-; CHECK: load
define void @load_fn_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_readonly() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -79,11 +101,15 @@ define void @load_fn_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_writeonly
-; CHECK: load
-; CHECK: call void @fn_writeonly()
-; CHECK: load
define void @load_fn_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_writeonly() #[[ATTR3:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -92,10 +118,15 @@ define void @load_fn_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_readnone
-; CHECK-DAG: load <2 x i32>
-; CHECK-DAG: call void @fn_readnone()
define void @load_fn_readnone(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_readnone(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: call void @fn_readnone() #[[ATTR5:[0-9]+]]
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -108,11 +139,15 @@ define void @load_fn_readnone(ptr %p) #0 {
; Same tests, but now for stores instead of loads.
; ------------------------------------------------
-; CHECK-LABEL: @store_fn
-; CHECK: store
-; CHECK: call void @fn()
-; CHECK: store
define void @store_fn(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -121,11 +156,15 @@ define void @store_fn(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind
-; CHECK: store
-; CHECK: call void @fn_nounwind()
-; CHECK: store
define void @store_fn_nounwind(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind() #[[ATTR0]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -134,11 +173,15 @@ define void @store_fn_nounwind(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind_writeonly
-; CHECK: store
-; CHECK: call void @fn_nounwind_writeonly()
-; CHECK: store
define void @store_fn_nounwind_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind_writeonly() #[[ATTR1]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -147,11 +190,15 @@ define void @store_fn_nounwind_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind_readonly
-; CHECK: store
-; CHECK: call void @fn_nounwind_readonly()
-; CHECK: store
define void @store_fn_nounwind_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind_readonly() #[[ATTR2]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -160,11 +207,15 @@ define void @store_fn_nounwind_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_readonly
-; CHECK: store
-; CHECK: call void @fn_readonly
-; CHECK: store
define void @store_fn_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_readonly() #[[ATTR4]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -173,11 +224,15 @@ define void @store_fn_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_writeonly
-; CHECK: store
-; CHECK: call void @fn_writeonly()
-; CHECK: store
define void @store_fn_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_writeonly() #[[ATTR3]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -187,10 +242,13 @@ define void @store_fn_writeonly(ptr %p) #0 {
}
; This is the only store idiom we can vectorize.
-; CHECK-LABEL: @store_fn_readnone
-; CHECK-DAG: store <2 x i32>
-; CHECK-DAG: call void @fn_readnone()
define void @store_fn_readnone(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_readnone(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @fn_readnone() #[[ATTR5]]
+; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p, align 8
@@ -207,3 +265,245 @@ attributes #3 = { writeonly }
attributes #4 = { readonly }
; readnone implies nounwind, so no need to test separately
attributes #5 = { nounwind willreturn readnone }
+
+; --------------------------------------------------------
+; Load chains across atomics.
+; --------------------------------------------------------
+; Vectorizing a load chain hoists its later loads up to the first one.
+; Hoisting a load above an intervening acquire operation would break
+; release/acquire synchronization, so ordered atomics conservatively act as
+; reordering barriers; only unordered loads may be crossed.
+
+; Hoisting the load of %p.1 above the acquire load would break the
+; release/acquire message-passing idiom.
+
+define <2 x i32> @load_acquire(ptr %p, ptr %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_acquire(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] acquire, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag acquire, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Monotonic imposes no ordering on other locations, but LSV is conservative.
+
+define <2 x i32> @load_monotonic(ptr %p, ptr %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_monotonic(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] monotonic, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag monotonic, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Unordered loads can be crossed.
+
+define <2 x i32> @load_unordered(ptr %p, ptr %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_unordered(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] unordered, align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V01]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag unordered, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A volatile load already terminates the region LSV vectorizes within.
+
+define <2 x i32> @load_volatile(ptr %p, ptr %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_volatile(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load volatile i32, ptr [[FLAG]], align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load volatile i32, ptr %flag, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A release store permits hoisting loads in principle; LSV is conservative.
+
+define <2 x i32> @load_release_store(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_release_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] release, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ store atomic i32 1, ptr %flag release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Same for RMWs...
+
+define <2 x i32> @load_release_rmw(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_release_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 release, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; ...and cmpxchg.
+
+define <2 x i32> @load_release_cmpxchg(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define <2 x i32> @load_release_cmpxchg(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = cmpxchg ptr [[FLAG]], i32 0, i32 1 release monotonic, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = cmpxchg ptr %flag, i32 0, i32 1 release monotonic, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; --------------------------------------------------------
+; Store chains across atomics.
+; --------------------------------------------------------
+; The mirror image: vectorizing a store chain sinks its earlier stores down
+; to the last one, so again ordered atomics act as barriers.
+
+; An acquire load permits sinking stores in principle; LSV is conservative.
+
+define void @store_acquire_load(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define void @store_acquire_load(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] acquire, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %f = load atomic i32, ptr %flag acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; Same for RMWs.
+
+define void @store_acquire_rmw(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define void @store_acquire_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 acquire, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; Sinking the first store below the release store would break release
+; semantics.
+
+define void @store_release_store(ptr %p, ptr noalias %flag) {
+;
+; CHECK-LABEL: define void @store_release_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] release, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ store atomic i32 1, ptr %flag release, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
More information about the llvm-commits
mailing list