[llvm] [LSV] Respect atomic ordering in isSafeToMove (PR #208631)
Justin Lebar via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 11:48:04 PDT 2026
https://github.com/jlebar updated https://github.com/llvm/llvm-project/pull/208631
>From 2455561a53580ee433a3fc7afd6a1684a679f908 Mon Sep 17 00:00:00 2001
From: Justin Lebar <justin.lebar at gmail.com>
Date: Sat, 18 Jul 2026 20:35:55 +0000
Subject: [PATCH] [LSV] Respect atomic ordering in isSafeToMove
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The LoadStoreVectorizer emits a merged load at the chain's first
element, hoisting every later element up to that point. isSafeToMove
skipped all loads in the hoisted-over region on the grounds that loads
reorder freely with loads — true only of unordered loads. An acquire
load is a one-way barrier: no later access may be hoisted above it,
including loads of unrelated locations.
define i32 @consume(ptr %data, ptr %flag) {
%lo = load i32, ptr %data, align 8
%f = load atomic i32, ptr %flag acquire, align 4
%p1 = getelementptr inbounds i8, ptr %data, i64 4
%hi = load i32, ptr %p1, align 4
...
}
$ opt -passes=load-store-vectorizer -mtriple=x86_64
%1 = load <2 x i32>, ptr %data, align 8 ; both words above the acquire
%f = load atomic i32, ptr %flag acquire, align 4
This breaks message passing: a producer stores data[1] = 2 and then
release-stores flag = 1; a consumer that observes %f == 1 is guaranteed
%hi == 2, but the rewritten code reads data[1] without waiting for the
acquire, so %hi can be stale.
The barrier is direction-aware, and vectorizing moves a chain in only
one direction: a load chain hoists (merged load at the first element),
a store chain sinks (merged store at the last). Per llvm/docs/Atomics.md
the hoist is exactly what release permits ("it is possible to move loads
from after a Release store or read-modify-write operation to before it")
and the sink what acquire permits ("it is possible to move stores from
before an Acquire load or read-modify-write operation to after it");
seq_cst allows "the same reorderings" except against other seq_cst
operations, and chain elements are simple. So a load chain splits at any
atomic with acquire semantics (acquire/seq_cst load; acquire or stronger
RMW or cmpxchg, counting the stronger of the cmpxchg's orderings) and a
store chain at any with release semantics, while an atomic that only
orders the other direction splits the chain only if it actually accesses
a chain location — asked of AA directly, since getModRefInfo stays
conservative about ordered atomics. Monotonic and unordered atomics are
never ordering barriers; volatile accesses and fences keep the
conservative treatment.
The barrier fall-through relies on getModRefInfo being conservative
about ordered atomics; the one gap there (it ignored a cmpxchg's
failure ordering) is fixed separately in "[AA] Use the cmpxchg merged
ordering in getModRefInfo".
---
.../Vectorize/LoadStoreVectorizer.cpp | 51 +-
.../NVPTX/merge-across-side-effects.ll | 534 ++++++++++++++++--
2 files changed, 527 insertions(+), 58 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 897285f722089..ac0bbb3e20047 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -96,6 +96,7 @@
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Alignment.h"
+#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/KnownBits.h"
@@ -1258,6 +1259,30 @@ bool Vectorizer::vectorizeChain(Chain &C) {
return true;
}
+// True if I's atomic ordering forbids the one motion vectorizing performs:
+// hoisting later loads up to the chain's first element (IsLoadChain) or
+// sinking earlier stores down to its last. Per llvm/docs/Atomics.md only
+// acquire semantics forbid the hoist and only release semantics the sink;
+// acquire attaches to an operation's load component and release to its store
+// component, and a cmpxchg's failure ordering can exceed its success
+// ordering.
+static bool isOrderingBarrier(const Instruction *I, bool IsLoadChain) {
+ assert(I->isAtomic() && !isa<FenceInst>(I));
+ if (IsLoadChain ? !I->hasAtomicLoad() : !I->hasAtomicStore())
+ return false;
+ AtomicOrdering Ordering;
+ if (auto *CX = dyn_cast<AtomicCmpXchgInst>(I))
+ Ordering = CX->getMergedOrdering();
+ else if (auto *LI = dyn_cast<LoadInst>(I))
+ Ordering = LI->getOrdering();
+ else if (auto *SI = dyn_cast<StoreInst>(I))
+ Ordering = SI->getOrdering();
+ else
+ Ordering = cast<AtomicRMWInst>(I)->getOrdering();
+ return IsLoadChain ? isAcquireOrStronger(Ordering)
+ : isReleaseOrStronger(Ordering);
+}
+
template <bool IsLoadChain>
bool Vectorizer::isSafeToMove(
Instruction *ChainElem, Instruction *ChainBegin,
@@ -1291,6 +1316,7 @@ bool Vectorizer::isSafeToMove(
const APInt &ChainElemOffset = ChainOffsets.at(ChainElem);
const unsigned ChainElemSize =
DL.getTypeStoreSize(getLoadStoreType(ChainElem));
+ const MemoryLocation ChainElemLoc = MemoryLocation::get(ChainElem);
for (; BBIt != BBItEnd; ++BBIt) {
Instruction *I = &*BBIt;
@@ -1298,14 +1324,30 @@ bool Vectorizer::isSafeToMove(
if (!I->mayReadOrWriteMemory())
continue;
- // Loads can be reordered with other loads.
- if (IsLoadChain && isa<LoadInst>(I))
+ // A load only reads, and a read can't clobber the chain's loads, so it
+ // limits a load chain only through its ordering: acquire or stronger
+ // forbids the hoist.
+ if (auto *LI = dyn_cast<LoadInst>(I);
+ IsLoadChain && LI && !LI->isVolatile() &&
+ !isAcquireOrStronger(LI->getOrdering()))
continue;
// Stores can be sunk below invariant loads.
if (!IsLoadChain && isInvariantLoad(I))
continue;
+ // An intervening atomic whose ordering permits our direction of motion
+ // (see isOrderingBarrier) limits us only through data dependence on the
+ // location it accesses. Check that directly: getModRefInfo can't, since
+ // it doesn't know which way we move ChainElem and so must treat a
+ // captured object as synchronized-with in both directions. Everything
+ // else -- ordering barriers, volatile atomics, fences -- keeps the
+ // conservative treatment below.
+ if (I->isAtomic() && !isa<FenceInst>(I) && !I->isVolatile() &&
+ !isOrderingBarrier(I, IsLoadChain) &&
+ BatchAA.isNoAlias(MemoryLocation::get(I), ChainElemLoc))
+ continue;
+
// If I is in the chain, we can tell whether it aliases ChainIt by checking
// what offset ChainIt accesses. This may be better than AA is able to do.
//
@@ -1329,8 +1371,7 @@ bool Vectorizer::isSafeToMove(
LLVM_DEBUG({
// Double check that AA also sees this alias. If not, we probably
// have a bug.
- ModRefInfo MR =
- BatchAA.getModRefInfo(I, MemoryLocation::get(ChainElem));
+ ModRefInfo MR = BatchAA.getModRefInfo(I, ChainElemLoc);
assert(IsLoadChain ? isModSet(MR) : isModOrRefSet(MR));
dbgs() << "LSV: Found alias in chain: " << *I << "\n";
});
@@ -1341,7 +1382,7 @@ bool Vectorizer::isSafeToMove(
}
LLVM_DEBUG(dbgs() << "LSV: Querying AA for " << *I << "\n");
- ModRefInfo MR = BatchAA.getModRefInfo(I, MemoryLocation::get(ChainElem));
+ ModRefInfo MR = BatchAA.getModRefInfo(I, ChainElemLoc);
if (IsLoadChain ? isModSet(MR) : isModOrRefSet(MR)) {
LLVM_DEBUG(dbgs() << "LSV: Found alias in chain:\n"
<< " Aliasing instruction:\n"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
index 782dba25f063d..a211c4f2e6532 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Check that the load/store vectorizer is willing to move loads/stores across
@@ -15,11 +16,15 @@ declare void @fn_writeonly() #3
declare void @fn_readonly() #4
declare void @fn_readnone() #5
-; CHECK-LABEL: @load_fn
-; CHECK: load
-; CHECK: call void @fn()
-; CHECK: load
define void @load_fn(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -28,11 +33,15 @@ define void @load_fn(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind
-; CHECK: load
-; CHECK: call void @fn_nounwind()
-; CHECK: load
define void @load_fn_nounwind(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_nounwind() #[[ATTR0]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -41,11 +50,15 @@ define void @load_fn_nounwind(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind_writeonly
-; CHECK: load
-; CHECK: call void @fn_nounwind_writeonly()
-; CHECK: load
define void @load_fn_nounwind_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_nounwind_writeonly() #[[ATTR1:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -54,10 +67,15 @@ define void @load_fn_nounwind_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind_readonly
-; CHECK-DAG: load <2 x i32>
-; CHECK-DAG: call void @fn_nounwind_readonly()
define void @load_fn_nounwind_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: call void @fn_nounwind_readonly() #[[ATTR2:[0-9]+]]
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -66,11 +84,15 @@ define void @load_fn_nounwind_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_readonly
-; CHECK: load
-; CHECK: call void @fn_readonly
-; CHECK: load
define void @load_fn_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_readonly() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -79,11 +101,15 @@ define void @load_fn_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_writeonly
-; CHECK: load
-; CHECK: call void @fn_writeonly()
-; CHECK: load
define void @load_fn_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_writeonly() #[[ATTR3:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -92,10 +118,15 @@ define void @load_fn_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_readnone
-; CHECK-DAG: load <2 x i32>
-; CHECK-DAG: call void @fn_readnone()
define void @load_fn_readnone(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_readnone(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: call void @fn_readnone() #[[ATTR5:[0-9]+]]
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -108,11 +139,15 @@ define void @load_fn_readnone(ptr %p) #0 {
; Same tests, but now for stores instead of loads.
; ------------------------------------------------
-; CHECK-LABEL: @store_fn
-; CHECK: store
-; CHECK: call void @fn()
-; CHECK: store
define void @store_fn(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -121,11 +156,15 @@ define void @store_fn(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind
-; CHECK: store
-; CHECK: call void @fn_nounwind()
-; CHECK: store
define void @store_fn_nounwind(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind() #[[ATTR0]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -134,11 +173,15 @@ define void @store_fn_nounwind(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind_writeonly
-; CHECK: store
-; CHECK: call void @fn_nounwind_writeonly()
-; CHECK: store
define void @store_fn_nounwind_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind_writeonly() #[[ATTR1]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -147,11 +190,15 @@ define void @store_fn_nounwind_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind_readonly
-; CHECK: store
-; CHECK: call void @fn_nounwind_readonly()
-; CHECK: store
define void @store_fn_nounwind_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind_readonly() #[[ATTR2]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -160,11 +207,15 @@ define void @store_fn_nounwind_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_readonly
-; CHECK: store
-; CHECK: call void @fn_readonly
-; CHECK: store
define void @store_fn_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_readonly() #[[ATTR4]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -173,11 +224,15 @@ define void @store_fn_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_writeonly
-; CHECK: store
-; CHECK: call void @fn_writeonly()
-; CHECK: store
define void @store_fn_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_writeonly() #[[ATTR3]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -187,10 +242,13 @@ define void @store_fn_writeonly(ptr %p) #0 {
}
; This is the only store idiom we can vectorize.
-; CHECK-LABEL: @store_fn_readnone
-; CHECK-DAG: store <2 x i32>
-; CHECK-DAG: call void @fn_readnone()
define void @store_fn_readnone(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_readnone(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @fn_readnone() #[[ATTR5]]
+; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p, align 8
@@ -207,3 +265,373 @@ attributes #3 = { writeonly }
attributes #4 = { readonly }
; readnone implies nounwind, so no need to test separately
attributes #5 = { nounwind willreturn readnone }
+
+; --------------------------------------------------------
+; Load chains across atomics.
+; --------------------------------------------------------
+; Vectorizing a load chain merges it at the position of its first element,
+; hoisting the later loads up. Per llvm/docs/Atomics.md that motion is
+; forbidden by acquire semantics but not by release semantics, so an
+; acquire operation always splits the chain, while a release-only operation
+; splits it only if it writes a location the chain reads.
+
+; Loads are not merged across an ordered atomic load: the load of %p.1 would
+; be hoisted above the acquire load of %flag, breaking the release/acquire
+; message-passing idiom.
+
+define <2 x i32> @load_acquire(ptr %p, ptr %flag) {
+; CHECK-LABEL: define <2 x i32> @load_acquire(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] acquire, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag acquire, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+
+; A monotonic load imposes no ordering on other locations, and a read can't
+; clobber the chain's loads, so this vectorizes with no alias query at all.
+
+define <2 x i32> @load_monotonic(ptr %p, ptr %flag) {
+; CHECK-LABEL: define <2 x i32> @load_monotonic(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] monotonic, align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V01]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag monotonic, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Unordered loads don't order anything; still vectorizes.
+
+define <2 x i32> @load_unordered(ptr %p, ptr %flag) {
+; CHECK-LABEL: define <2 x i32> @load_unordered(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] unordered, align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V01]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load atomic i32, ptr %flag unordered, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Unchanged by this patch: a volatile load is not guaranteed to transfer
+; execution to its successor, so it already terminates the region LSV
+; vectorizes within.
+
+define <2 x i32> @load_volatile(ptr %p, ptr %flag) {
+; CHECK-LABEL: define <2 x i32> @load_volatile(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load volatile i32, ptr [[FLAG]], align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[V0]], [[F]]
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %f = load volatile i32, ptr %flag, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %add = add i32 %v0, %f
+ %r0 = insertelement <2 x i32> poison, i32 %add, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A release store doesn't forbid hoisting loads above it, so with AA proving
+; no-alias this vectorizes.
+
+define <2 x i32> @load_release_store(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_release_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] release, align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V01]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ store atomic i32 1, ptr %flag release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; Against the chain's simple loads a seq_cst store acts like a release store
+; (seq_cst adds constraints only against other seq_cst operations), so this
+; vectorizes too.
+
+define <2 x i32> @load_seq_cst_store(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_seq_cst_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] seq_cst, align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V01]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ store atomic i32 1, ptr %flag seq_cst, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; A release RMW has no acquire semantics either. It also reads %flag, but a
+; read can't clobber the chain's loads; only its write matters, and AA proves
+; it doesn't alias.
+
+define <2 x i32> @load_release_rmw(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_release_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 release, align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V01]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; cmpxchg release/monotonic likewise has no acquire semantics; vectorizes.
+
+define <2 x i32> @load_release_cmpxchg(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_release_cmpxchg(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[OLD:%.*]] = cmpxchg ptr [[FLAG]], i32 0, i32 1 release monotonic, align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V01]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V12]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = cmpxchg ptr %flag, i32 0, i32 1 release monotonic, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; An acq_rel RMW has acquire semantics, so it's a barrier regardless of
+; aliasing.
+
+define <2 x i32> @load_acq_rel_rmw(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_acq_rel_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 acq_rel, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 acq_rel, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; The cmpxchg failure ordering counts too: success is release-only, but
+; failure is acquire, so hoisting loads above the cmpxchg is forbidden.
+
+define <2 x i32> @load_release_acquire_cmpxchg(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define <2 x i32> @load_release_acquire_cmpxchg(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[OLD:%.*]] = cmpxchg ptr [[FLAG]], i32 0, i32 1 release acquire, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ %old = cmpxchg ptr %flag, i32 0, i32 1 release acquire, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; The ordering of a release store permits the hoist, but data dependence
+; doesn't: it writes a location the chain loads.
+
+define <2 x i32> @load_aliasing_release_store(ptr %p) {
+; CHECK-LABEL: define <2 x i32> @load_aliasing_release_store(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: store atomic i32 1, ptr [[P_1]] release, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[V0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[V1]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[R1]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ %v0 = load i32, ptr %p, align 8
+ store atomic i32 1, ptr %p.1 release, align 4
+ %v1 = load i32, ptr %p.1, align 4
+ %r0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %r1 = insertelement <2 x i32> %r0, i32 %v1, i32 1
+ ret <2 x i32> %r1
+}
+
+; --------------------------------------------------------
+; Store chains across atomics.
+; --------------------------------------------------------
+; Vectorizing a store chain merges it at the position of its last element,
+; sinking the earlier stores down. That's the mirror image: release
+; semantics always split the chain, while an acquire-only operation splits
+; it only if it accesses a location the chain writes.
+
+; An acquire load doesn't forbid sinking stores below it; AA proves no-alias,
+; so this vectorizes.
+
+define void @store_acquire_load(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_acquire_load(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] acquire, align 4
+; CHECK-NEXT: store <2 x i32> <i32 0, i32 1>, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %f = load atomic i32, ptr %flag acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; Against the chain's simple stores a seq_cst load acts like an acquire load;
+; vectorizes.
+
+define void @store_seq_cst_load(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_seq_cst_load(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[FLAG]] seq_cst, align 4
+; CHECK-NEXT: store <2 x i32> <i32 0, i32 1>, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %f = load atomic i32, ptr %flag seq_cst, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; An acquire RMW has no release semantics; its read and write both stay ahead
+; of the merged store, and AA proves they can't touch the chain's locations.
+
+define void @store_acquire_rmw(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_acquire_rmw(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[FLAG]], i32 1 acquire, align 4
+; CHECK-NEXT: store <2 x i32> <i32 0, i32 1>, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %old = atomicrmw add ptr %flag, i32 1 acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+; A release store is a barrier for sinking stores, regardless of aliasing.
+
+define void @store_release_store(ptr %p, ptr noalias %flag) {
+; CHECK-LABEL: define void @store_release_store(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[FLAG:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: store atomic i32 1, ptr [[FLAG]] release, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ store atomic i32 1, ptr %flag release, align 4
+ store i32 1, ptr %p.1, align 4
+ ret void
+}
+
+
+; An acquire load reading a chain store's location: the ordering permits the
+; sink, but data dependence doesn't -- the load must observe the first store.
+
+define i32 @store_aliasing_acquire_load(ptr %p) {
+; CHECK-LABEL: define i32 @store_aliasing_acquire_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: store i32 0, ptr [[P]], align 8
+; CHECK-NEXT: [[F:%.*]] = load atomic i32, ptr [[P]] acquire, align 4
+; CHECK-NEXT: store i32 1, ptr [[P_1]], align 4
+; CHECK-NEXT: ret i32 [[F]]
+;
+ %p.1 = getelementptr i8, ptr %p, i64 4
+ store i32 0, ptr %p, align 8
+ %f = load atomic i32, ptr %p acquire, align 4
+ store i32 1, ptr %p.1, align 4
+ ret i32 %f
+}
More information about the llvm-commits
mailing list