[llvm] [VPlan] Handle AtomicRMW/AtomicCmpXchg/Fence in outer loop creation. (PR #209031)

Florian Hahn via llvm-commits llvm-commits at lists.llvm.org
Sun Jul 12 13:12:46 PDT 2026


https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/209031

>From ac5987b642d046cc0431299226ddadcdc7dce723 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 20 Jun 2026 21:04:46 +0100
Subject: [PATCH] [VPlan] Handle AtomicRMW/AtomicCmpXchg/Fence in outer loop
 creation.

Teach VPInstruction::getNumOperandsForOpcode about AtomicRMW, AtomicCmpXchg
and Fence, so VPlan construction over an outer loop that still contains
these operations does not crash before later legality rejects the loop.

Fixes crashes in the added tests.
---
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |   3 +
 .../Transforms/Vectorize/VPlanTransforms.cpp  |   5 +
 .../LoopVectorize/outer-loop-atomic.ll        | 168 ++++++++++++++++++
 3 files changed, 176 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/outer-loop-atomic.ll

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fa67ac7246470..3fbeb7e772a2e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -670,6 +670,9 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
   case Instruction::GetElementPtr:
   case Instruction::PHI:
   case Instruction::Switch:
+  case Instruction::AtomicRMW:
+  case Instruction::AtomicCmpXchg:
+  case Instruction::Fence:
   case VPInstruction::AnyOf:
   case VPInstruction::BuildStructVector:
   case VPInstruction::BuildVector:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f8198cf9133bf..c526f95311d5f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -69,6 +69,11 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
 
       Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
 
+      // Atomic accesses and fences have ordering/atomicity semantics that
+      // cannot be preserved by lane-wise widening.
+      if (isa<AtomicRMWInst, AtomicCmpXchgInst, FenceInst>(Inst))
+        return false;
+
       VPRecipeBase *NewRecipe = nullptr;
       if (auto *PhiR = dyn_cast<VPPhi>(&Ingredient)) {
         auto *Phi = cast<PHINode>(PhiR->getUnderlyingValue());
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-atomic.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-atomic.ll
new file mode 100644
index 0000000000000..6e19f71736544
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-atomic.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
+
+define void @unsafe_atomicrmw(ptr noalias %A, ptr %ctr, i64 %N, i64 %M) {
+; CHECK-LABEL: define void @unsafe_atomicrmw(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[CTR:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[OLD:%.*]] = atomicrmw add ptr [[CTR]], i32 1 seq_cst, align 4
+; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
+; CHECK:       [[INNER_BODY]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
+; CHECK-NEXT:    [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
+; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
+; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
+; CHECK-NEXT:    [[V:%.*]] = trunc i64 [[I]] to i32
+; CHECK-NEXT:    store i32 [[V]], ptr [[A_PTR]], align 4
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT:    [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
+; CHECK-NEXT:    br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  %old = atomicrmw add ptr %ctr, i32 1 seq_cst, align 4
+  br label %inner.body
+
+inner.body:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
+  %i.mul.M = mul nsw i64 %i, %M
+  %idx = add nsw i64 %i.mul.M, %j
+  %A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
+  %v = trunc i64 %i to i32
+  store i32 %v, ptr %A.ptr, align 4
+  %j.next = add nuw nsw i64 %j, 1
+  %j.cmp = icmp eq i64 %j.next, %M
+  br i1 %j.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %i.cmp = icmp eq i64 %i.next, %N
+  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+define void @unsafe_cmpxchg(ptr noalias %A, ptr %ctr, i64 %N, i64 %M) {
+; CHECK-LABEL: define void @unsafe_cmpxchg(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[CTR:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[PAIR:%.*]] = cmpxchg ptr [[CTR]], i32 0, i32 1 seq_cst seq_cst, align 4
+; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
+; CHECK:       [[INNER_BODY]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
+; CHECK-NEXT:    [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
+; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
+; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
+; CHECK-NEXT:    [[V:%.*]] = trunc i64 [[I]] to i32
+; CHECK-NEXT:    store i32 [[V]], ptr [[A_PTR]], align 4
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT:    [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
+; CHECK-NEXT:    br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  %pair = cmpxchg ptr %ctr, i32 0, i32 1 seq_cst seq_cst, align 4
+  br label %inner.body
+
+inner.body:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
+  %i.mul.M = mul nsw i64 %i, %M
+  %idx = add nsw i64 %i.mul.M, %j
+  %A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
+  %v = trunc i64 %i to i32
+  store i32 %v, ptr %A.ptr, align 4
+  %j.next = add nuw nsw i64 %j, 1
+  %j.cmp = icmp eq i64 %j.next, %M
+  br i1 %j.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %i.cmp = icmp eq i64 %i.next, %N
+  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+define void @unsafe_fence(ptr noalias %A, i64 %N, i64 %M) {
+; CHECK-LABEL: define void @unsafe_fence(
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
+; CHECK:       [[INNER_BODY]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
+; CHECK-NEXT:    [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
+; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
+; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
+; CHECK-NEXT:    [[V:%.*]] = trunc i64 [[I]] to i32
+; CHECK-NEXT:    store i32 [[V]], ptr [[A_PTR]], align 4
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT:    [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
+; CHECK-NEXT:    br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  fence seq_cst
+  br label %inner.body
+
+inner.body:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
+  %i.mul.M = mul nsw i64 %i, %M
+  %idx = add nsw i64 %i.mul.M, %j
+  %A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
+  %v = trunc i64 %i to i32
+  store i32 %v, ptr %A.ptr, align 4
+  %j.next = add nuw nsw i64 %j, 1
+  %j.cmp = icmp eq i64 %j.next, %M
+  br i1 %j.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %i.cmp = icmp eq i64 %i.next, %N
+  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.vectorize.width", i32 4}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}



More information about the llvm-commits mailing list