[llvm] [LSV] Respect atomic ordering in isSafeToMove (PR #208631)
Justin Lebar via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 18 13:58:09 PDT 2026
https://github.com/jlebar updated https://github.com/llvm/llvm-project/pull/208631
>From 3d493062522fe663426208bfe2b59c9982cdb9fb Mon Sep 17 00:00:00 2001
From: Justin Lebar <justin.lebar at gmail.com>
Date: Sat, 18 Jul 2026 20:35:55 +0000
Subject: [PATCH] [LSV] Respect atomic ordering in isSafeToMove
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The LoadStoreVectorizer emits a merged load at the chain's first
element, hoisting every later element up to that point. isSafeToMove
skipped all loads in the hoisted-over region on the grounds that loads
reorder freely with loads — true only of unordered loads. An acquire
load is a one-way barrier: no later access may be hoisted above it,
including loads of unrelated locations.
define i32 @consume(ptr %data, ptr %flag) {
%lo = load i32, ptr %data, align 8
%f = load atomic i32, ptr %flag acquire, align 4
%p1 = getelementptr inbounds i8, ptr %data, i64 4
%hi = load i32, ptr %p1, align 4
...
}
$ opt -passes=load-store-vectorizer -mtriple=x86_64
%1 = load <2 x i32>, ptr %data, align 8 ; both words above the acquire
%f = load atomic i32, ptr %flag acquire, align 4
This breaks message passing: a producer stores data[1] = 2 and then
release-stores flag = 1; a consumer that observes %f == 1 is guaranteed
%hi == 2, but the rewritten code reads data[1] without waiting for the
acquire, so %hi can be stale.
The barrier is direction-aware, and vectorizing moves a chain in only
one direction: a load chain hoists (merged load at the first element),
a store chain sinks (merged store at the last). Per llvm/docs/Atomics.md
the hoist is exactly what release permits ("it is possible to move loads
from after a Release store or read-modify-write operation to before it")
and the sink what acquire permits ("it is possible to move stores from
before an Acquire load or read-modify-write operation to after it");
seq_cst allows "the same reorderings" except against other seq_cst
operations, and chain elements are simple. So a load chain splits at any
atomic with acquire semantics (acquire/seq_cst load; acquire or stronger
RMW or cmpxchg, counting the stronger of the cmpxchg's orderings) and a
store chain at any with release semantics, while an atomic that only
orders the other direction splits the chain only if it actually accesses
a chain location — asked of AA directly, since getModRefInfo stays
conservative about ordered atomics. Monotonic and unordered atomics are
never ordering barriers; volatile accesses and fences keep the
conservative treatment.
The barrier fall-through relies on getModRefInfo being conservative
about ordered atomics; the one gap there (it ignored a cmpxchg's
failure ordering) is fixed separately in "[AA] Use the cmpxchg merged
ordering in getModRefInfo".
---
.../Vectorize/LoadStoreVectorizer.cpp | 56 ++-
.../NVPTX/merge-across-side-effects.ll | 411 ++++++++++++++++++
2 files changed, 462 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index b5437ad8705bf..6bc1b8a211b2a 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -96,6 +96,7 @@
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Alignment.h"
+#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/KnownBits.h"
@@ -1260,6 +1261,29 @@ bool Vectorizer::vectorizeChain(Chain &C) {
return true;
}
+// True if I's atomic ordering forbids the one motion vectorizing performs:
+// hoisting later loads up to the chain's first element (IsLoadChain) or
+// sinking earlier stores down to its last. Per llvm/docs/Atomics.md only
+// acquire semantics forbid the hoist and only release semantics the sink;
+// acquire attaches to an operation's load component and release to its store
+// component, and a cmpxchg's failure ordering can exceed its success
+// ordering.
+static bool isOrderingBarrier(const Instruction *I, bool IsLoadChain) {
+ if (IsLoadChain ? !I->hasAtomicLoad() : !I->hasAtomicStore())
+ return false;
+ AtomicOrdering Ordering;
+ if (auto *CX = dyn_cast<AtomicCmpXchgInst>(I))
+ Ordering = CX->getMergedOrdering();
+ else if (auto *LI = dyn_cast<LoadInst>(I))
+ Ordering = LI->getOrdering();
+ else if (auto *SI = dyn_cast<StoreInst>(I))
+ Ordering = SI->getOrdering();
+ else
+ Ordering = cast<AtomicRMWInst>(I)->getOrdering();
+ return IsLoadChain ? isAcquireOrStronger(Ordering)
+ : isReleaseOrStronger(Ordering);
+}
+
template <bool IsLoadChain>
bool Vectorizer::isSafeToMove(
Instruction *ChainElem, Instruction *ChainBegin,
@@ -1293,6 +1317,7 @@ bool Vectorizer::isSafeToMove(
const APInt &ChainElemOffset = ChainOffsets.at(ChainElem);
const unsigned ChainElemSize =
DL.getTypeStoreSize(getLoadStoreType(ChainElem));
+ const MemoryLocation ChainElemLoc = MemoryLocation::get(ChainElem);
for (; BBIt != BBItEnd; ++BBIt) {
Instruction *I = &*BBIt;
@@ -1300,14 +1325,36 @@ bool Vectorizer::isSafeToMove(
if (!I->mayReadOrWriteMemory())
continue;
- // Loads can be reordered with other loads.
- if (IsLoadChain && isa<LoadInst>(I))
+ // Loads can be reordered with other unordered loads. Ordered loads are
+ // handled below.
+ if (auto *LI = dyn_cast<LoadInst>(I);
+ IsLoadChain && LI && LI->isUnordered())
continue;
// Stores can be sunk below invariant loads.
if (!IsLoadChain && isInvariantLoad(I))
continue;
+ // An intervening atomic whose ordering permits our direction of motion
+ // (see isOrderingBarrier) limits us only through data dependence on the
+ // location it accesses, which we must check directly: getModRefInfo is
+ // conservative about any stronger-than-monotonic atomic. Volatile
+ // atomics and fences keep the conservative treatment below.
+ if (I->isAtomic() && !isa<FenceInst>(I) && !I->isVolatile()) {
+ if (!isOrderingBarrier(I, IsLoadChain)) {
+ // An atomic load only reads, and a read can't clobber the chain's
+ // loads.
+ if (IsLoadChain && isa<LoadInst>(I))
+ continue;
+ if (BatchAA.isNoAlias(MemoryLocation::get(I), ChainElemLoc))
+ continue;
+ LLVM_DEBUG(dbgs() << "LSV: Atomic aliases chain: " << *I << "\n");
+ return false;
+ }
+ // Fall through: getModRefInfo can still prove the reordering
+ // unobservable (e.g. the chain's underlying object is never captured).
+ }
+
// If I is in the chain, we can tell whether it aliases ChainIt by checking
// what offset ChainIt accesses. This may be better than AA is able to do.
//
@@ -1331,8 +1378,7 @@ bool Vectorizer::isSafeToMove(
LLVM_DEBUG({
// Double check that AA also sees this alias. If not, we probably
// have a bug.
- ModRefInfo MR =
- BatchAA.getModRefInfo(I, MemoryLocation::get(ChainElem));
+ ModRefInfo MR = BatchAA.getModRefInfo(I, ChainElemLoc);
assert(IsLoadChain ? isModSet(MR) : isModOrRefSet(MR));
dbgs() << "LSV: Found alias in chain: " << *I << "\n";
});
@@ -1343,7 +1389,7 @@ bool Vectorizer::isSafeToMove(
}
LLVM_DEBUG(dbgs() << "LSV: Querying AA for " << *I << "\n");
- ModRefInfo MR = BatchAA.getModRefInfo(I, MemoryLocation::get(ChainElem));
+ ModRefInfo MR = BatchAA.getModRefInfo(I, ChainElemLoc);
if (IsLoadChain ? isModSet(MR) : isModOrRefSet(MR)) {
LLVM_DEBUG(dbgs() << "LSV: Found alias in chain:\n"
<< " Aliasing instruction:\n"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
index 782dba25f063d..80032f5f91d3f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
@@ -207,3 +207,414 @@ attributes #3 = { writeonly }
attributes #4 = { readonly }
; readnone implies nounwind, so no need to test separately
attributes #5 = { nounwind willreturn readnone }
+
+; --------------------------------------------------------
+; Load chains across atomics.
+; --------------------------------------------------------
+; Vectorizing a load chain merges it at the position of its first element,
+; hoisting the later loads up. Per llvm/docs/Atomics.md that motion is
+; forbidden by acquire semantics but not by release semantics, so an
+; acquire operation always splits the chain, while a release-only operation
+; splits it only if it writes a location the chain reads.
+
+; Loads are not merged across an ordered atomic load: the load of %p.1 would
+; be hoisted above the acquire load of %flag, breaking the release/acquire
+; message-passing idiom.
+
+define <2 x i32> @load_acquire(ptr %p, ptr %flag) {
+; CHECK-LABEL: define <2 x i32> @load_acquire(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] acquire, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag acquire, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Even when AA proves the flag doesn't alias the chain, %p may be observed by
+; other threads, so the ordering alone blocks vectorization.
+
+define <2 x i32> @load_seq_cst(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_seq_cst(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] seq_cst, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag seq_cst, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A monotonic load imposes no ordering on other locations, and AA proves the
+; flag doesn't alias the chain, so this still vectorizes.
+
+define <2 x i32> @load_monotonic(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_monotonic(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] monotonic, align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V01]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag monotonic, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Unordered loads don't order anything; still vectorizes.
+
+define <2 x i32> @load_unordered(ptr %p, ptr %flag) {
+; CHECK-LABEL: define <2 x i32> @load_unordered(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] unordered, align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V01]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag unordered, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A volatile load is not guaranteed to transfer execution to its successor, so
+; it already terminates the region LSV vectorizes within; not vectorized,
+; before or after.
+
+define <2 x i32> @load_volatile(ptr %p, ptr %flag) {
+; CHECK-LABEL: define <2 x i32> @load_volatile(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load volatile i32, ptr [[FLAG]], align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load volatile i32, ptr %flag, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A release store doesn't forbid hoisting loads above it, so with AA proving
+; no-alias this vectorizes.
+
+define <2 x i32> @load_release_store(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_release_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] release, align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V01]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ store atomic i32 1, ptr %flag release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Against the chain's simple loads a seq_cst store acts like a release store
+; (seq_cst adds constraints only against other seq_cst operations), so this
+; vectorizes too.
+
+define <2 x i32> @load_seq_cst_store(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_seq_cst_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] seq_cst, align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V01]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ store atomic i32 1, ptr %flag seq_cst, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A release RMW has no acquire semantics either. It also reads %flag, but a
+; read can't clobber the chain's loads; only its write matters, and AA proves
+; it doesn't alias.
+
+define <2 x i32> @load_release_rmw(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_release_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 release, align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V01]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; cmpxchg release/monotonic likewise has no acquire semantics; vectorizes.
+
+define <2 x i32> @load_release_cmpxchg(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_release_cmpxchg(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[OLD:%.*]] = cmpxchg ptr [[FLAG]], i32 0, i32 1 release monotonic, align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V01]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = cmpxchg ptr %flag, i32 0, i32 1 release monotonic, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; An acq_rel RMW has acquire semantics, so it's a barrier regardless of
+; aliasing.
+
+define <2 x i32> @load_acq_rel_rmw(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_acq_rel_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 acq_rel, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 acq_rel, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; The cmpxchg failure ordering counts too: success is release-only, but
+; failure is acquire, so hoisting loads above the cmpxchg is forbidden.
+
+define <2 x i32> @load_release_acquire_cmpxchg(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_release_acquire_cmpxchg(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = cmpxchg ptr [[FLAG]], i32 0, i32 1 release acquire, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = cmpxchg ptr %flag, i32 0, i32 1 release acquire, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; The ordering of a release store permits the hoist, but data dependence
+; doesn't: it writes a location the chain loads.
+
+define <2 x i32> @load_aliasing_release_store(ptr %p) {
+; CHECK-LABEL: define <2 x i32> @load_aliasing_release_store(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: store atomic i32 1, ptr [[P_1]] release, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ store atomic i32 1, ptr %p.1 release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; --------------------------------------------------------
+; Store chains across atomics.
+; --------------------------------------------------------
+; Vectorizing a store chain merges it at the position of its last element,
+; sinking the earlier stores down. That's the mirror image: release
+; semantics always split the chain, while an acquire-only operation splits
+; it only if it accesses a location the chain writes.
+
+; An acquire load doesn't forbid sinking stores below it; AA proves no-alias,
+; so this vectorizes.
+
+define void @store_acquire_load(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_acquire_load(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] acquire, align 4
+; CHECK-NEXT: store <2 x i32> <i32 0, i32 1>, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %f = load atomic i32, ptr %flag acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; Against the chain's simple stores a seq_cst load acts like an acquire load;
+; vectorizes.
+
+define void @store_seq_cst_load(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_seq_cst_load(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] seq_cst, align 4
+; CHECK-NEXT: store <2 x i32> <i32 0, i32 1>, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %f = load atomic i32, ptr %flag seq_cst, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; An acquire RMW has no release semantics; its read and write both stay ahead
+; of the merged store, and AA proves they can't touch the chain's locations.
+
+define void @store_acquire_rmw(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_acquire_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 acquire, align 4
+; CHECK-NEXT: store <2 x i32> <i32 0, i32 1>, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; A release store is a barrier for sinking stores, regardless of aliasing.
+
+define void @store_release_store(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_release_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] release, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ store atomic i32 1, ptr %flag release, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; So is an acq_rel RMW.
+
+define void @store_acq_rel_rmw(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_acq_rel_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 acq_rel, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 acq_rel, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; An acquire load reading a chain store's location: the ordering permits the
+; sink, but data dependence doesn't -- the load must observe the first store.
+
+define i32 @store_aliasing_acquire_load(ptr %p) {
+; CHECK-LABEL: define i32 @store_aliasing_acquire_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[P]] acquire, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret i32 [[F]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %f = load atomic i32, ptr %p acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret i32 %f
+}
More information about the llvm-commits
mailing list