[llvm-branch-commits] [llvm] SystemZ: Lower sub-word atomicrmw during type legalization (PR #238532)

Matt Arsenault via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Thu Nov 5 11:46:05 PST 2026


https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/238532

>From c0f525047aebef9685db1bfae7715b365c69b56a Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 3 Nov 2026 13:11:29 -0800
Subject: [PATCH 1/2] SystemZ: Lower sub-word atomicrmw during type
 legalization

Produce the ATOMIC_LOADW_* nodes when replacing the illegal i8/i16
results, instead of custom lowering the promoted node later. This
removes the no-op custom type legalization case.

Constants are now any-extended, so one test materializes an immediate
with llill instead of lhi.

Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
 .../Target/SystemZ/SystemZISelLowering.cpp    | 114 +++++++++---------
 llvm/lib/Target/SystemZ/SystemZISelLowering.h |   4 +-
 .../test/CodeGen/SystemZ/atomicrmw-xchg-02.ll |   2 +-
 3 files changed, 63 insertions(+), 57 deletions(-)

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 87129b03044e7d..6b576fc3058e2d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -310,9 +310,8 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
   setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
 
-  // Type legalization will convert 8- and 16-bit atomic operations into
-  // forms that operate on i32s (but still keeping the original memory VT).
-  // Lower them into full i32 operations.
+  // Lower 8- and 16-bit atomic operations into full i32 operations during
+  // type legalization.
   setOperationAction(
       {ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD, ISD::ATOMIC_LOAD_SUB,
        ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR, ISD::ATOMIC_LOAD_XOR,
@@ -5290,19 +5289,52 @@ static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
 
 }
 
-// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation.  Lower the first
-// two into the fullword ATOMIC_LOADW_* operation given by Opcode.
-SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
-                                                   SelectionDAG &DAG,
-                                                   unsigned Opcode) const {
-  auto *Node = cast<AtomicSDNode>(Op.getNode());
+// N is an 8- or 16-bit ATOMIC_LOAD_* operation.  Lower it into the fullword
+// ATOMIC_LOADW_* operation.
+void SystemZTargetLowering::lowerATOMIC_LOAD_OP(
+    SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
+  auto *Node = cast<AtomicSDNode>(N);
+  unsigned Opcode;
+  switch (Node->getOpcode()) {
+  case ISD::ATOMIC_SWAP:
+    Opcode = SystemZISD::ATOMIC_SWAPW;
+    break;
+  case ISD::ATOMIC_LOAD_ADD:
+    Opcode = SystemZISD::ATOMIC_LOADW_ADD;
+    break;
+  case ISD::ATOMIC_LOAD_SUB:
+    Opcode = SystemZISD::ATOMIC_LOADW_SUB;
+    break;
+  case ISD::ATOMIC_LOAD_AND:
+    Opcode = SystemZISD::ATOMIC_LOADW_AND;
+    break;
+  case ISD::ATOMIC_LOAD_OR:
+    Opcode = SystemZISD::ATOMIC_LOADW_OR;
+    break;
+  case ISD::ATOMIC_LOAD_XOR:
+    Opcode = SystemZISD::ATOMIC_LOADW_XOR;
+    break;
+  case ISD::ATOMIC_LOAD_NAND:
+    Opcode = SystemZISD::ATOMIC_LOADW_NAND;
+    break;
+  case ISD::ATOMIC_LOAD_MIN:
+    Opcode = SystemZISD::ATOMIC_LOADW_MIN;
+    break;
+  case ISD::ATOMIC_LOAD_MAX:
+    Opcode = SystemZISD::ATOMIC_LOADW_MAX;
+    break;
+  case ISD::ATOMIC_LOAD_UMIN:
+    Opcode = SystemZISD::ATOMIC_LOADW_UMIN;
+    break;
+  case ISD::ATOMIC_LOAD_UMAX:
+    Opcode = SystemZISD::ATOMIC_LOADW_UMAX;
+    break;
+  default:
+    llvm_unreachable("Unexpected atomic operation");
+  }
 
-  // 32-bit operations need no special handling.
   EVT NarrowVT = Node->getMemoryVT();
   EVT WideVT = MVT::i32;
-  if (NarrowVT == WideVT)
-    return Op;
-
   int64_t BitSize = NarrowVT.getSizeInBits();
   SDValue ChainIn = Node->getChain();
   SDValue Addr = Node->getBasePtr();
@@ -5314,8 +5346,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
   if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
     if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
       Opcode = SystemZISD::ATOMIC_LOADW_ADD;
-      Src2 = DAG.getSignedConstant(-Const->getSExtValue(), DL,
-                                   Src2.getValueType());
+      Src2 = DAG.getConstant(-Const->getAPIntValue(), DL, Src2.getValueType());
     }
 
   SDValue AlignedAddr, BitShift, NegBitShift;
@@ -5326,6 +5357,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
   // operations require the source to be shifted in advance.  (This shift
   // can be folded if the source is constant.)  For AND and NAND, the lower
   // bits must be set, while for other opcodes they should be left clear.
+  Src2 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Src2);
   if (Opcode != SystemZISD::ATOMIC_SWAPW)
     Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
                        DAG.getConstant(32 - BitSize, DL, WideVT));
@@ -5346,32 +5378,25 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
   SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
                                     DAG.getConstant(BitSize, DL, WideVT));
   SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
-
-  SDValue RetOps[2] = { Result, AtomicOp.getValue(1) };
-  return DAG.getMergeValues(RetOps, DL);
+  Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Result));
+  Results.push_back(AtomicOp.getValue(1));
 }
 
-// Op is an ATOMIC_LOAD_SUB operation.  Lower 8- and 16-bit operations into
-// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
+// Op is a 32- or 64-bit ATOMIC_LOAD_SUB operation.  Convert it into an
+// addition.
 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
                                                     SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   EVT MemVT = Node->getMemoryVT();
-  if (MemVT == MVT::i32 || MemVT == MVT::i64) {
-    // A full-width operation: negate and use LAA(G).
-    assert(Op.getValueType() == MemVT && "Mismatched VTs");
-    assert(Subtarget.hasInterlockedAccess1() &&
-           "Should have been expanded by AtomicExpand pass.");
-    SDValue Src2 = Node->getVal();
-    SDLoc DL(Src2);
-    SDValue NegSrc2 =
+  assert(Op.getValueType() == MemVT && "Mismatched VTs");
+  assert(Subtarget.hasInterlockedAccess1() &&
+         "Should have been expanded by AtomicExpand pass.");
+  SDValue Src2 = Node->getVal();
+  SDLoc DL(Src2);
+  SDValue NegSrc2 =
       DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), Src2);
-    return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
-                         Node->getChain(), Node->getBasePtr(), NegSrc2,
-                         Node->getMemOperand());
-  }
-
-  return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB);
+  return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, Node->getChain(),
+                       Node->getBasePtr(), NegSrc2, Node->getMemOperand());
 }
 
 // Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
@@ -7340,32 +7365,12 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
     return lowerVECREDUCE_ADD(Op, DAG);
   case ISD::ATOMIC_FENCE:
     return lowerATOMIC_FENCE(Op, DAG);
-  case ISD::ATOMIC_SWAP:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
   case ISD::ATOMIC_STORE:
     return lowerATOMIC_STORE(Op, DAG);
   case ISD::ATOMIC_LOAD:
     return lowerATOMIC_LOAD(Op, DAG);
-  case ISD::ATOMIC_LOAD_ADD:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
   case ISD::ATOMIC_LOAD_SUB:
     return lowerATOMIC_LOAD_SUB(Op, DAG);
-  case ISD::ATOMIC_LOAD_AND:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND);
-  case ISD::ATOMIC_LOAD_OR:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR);
-  case ISD::ATOMIC_LOAD_XOR:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR);
-  case ISD::ATOMIC_LOAD_NAND:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND);
-  case ISD::ATOMIC_LOAD_MIN:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN);
-  case ISD::ATOMIC_LOAD_MAX:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX);
-  case ISD::ATOMIC_LOAD_UMIN:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
-  case ISD::ATOMIC_LOAD_UMAX:
-    return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
     return lowerATOMIC_CMP_SWAP(Op, DAG);
   case ISD::STACKSAVE:
@@ -7641,6 +7646,7 @@ SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
   case ISD::ATOMIC_LOAD_MAX:
   case ISD::ATOMIC_LOAD_UMIN:
   case ISD::ATOMIC_LOAD_UMAX:
+    lowerATOMIC_LOAD_OP(N, Results, DAG);
     break;
   default:
     llvm_unreachable("Unexpected node to lower");
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index d90a8b9c7ff095..c5bc2ee13a1f2b 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -358,8 +358,8 @@ class SystemZTargetLowering : public TargetLowering {
   SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LDST_I128(SDValue Op, SelectionDAG &DAG) const;
-  SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
-                              unsigned Opcode) const;
+  void lowerATOMIC_LOAD_OP(SDNode *N, SmallVectorImpl<SDValue> &Results,
+                           SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
index 2cc6b2b9aea8b0..7113f04e32e49e 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
@@ -44,7 +44,7 @@ define i16 @f1(ptr %src, i16 %b) {
 ; a register and use the sequence above.
 define i16 @f2(ptr %src) {
 ; CHECK-LABEL: f2:
-; CHECK: lhi [[VALUE:%r[0-9]+]], -25536
+; CHECK: llill [[VALUE:%r[0-9]+]], 40000
 ; CHECK: risbg {{%r[0-9]+}}, [[VALUE]], 32, 47, 16
 ; CHECK: br %r14
 ;

>From 1d32fdd842ba1650984ef1fc212c539fc5eb221e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 5 Nov 2026 11:44:28 -0800
Subject: [PATCH 2/2] Pass the opcode into lowerATOMIC_LOAD_OP

---
 .../Target/SystemZ/SystemZISelLowering.cpp    | 63 +++++++------------
 llvm/lib/Target/SystemZ/SystemZISelLowering.h |  2 +-
 2 files changed, 24 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 6b576fc3058e2d..ec10c04e723f6f 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -5292,46 +5292,9 @@ static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
 // N is an 8- or 16-bit ATOMIC_LOAD_* operation.  Lower it into the fullword
 // ATOMIC_LOADW_* operation.
 void SystemZTargetLowering::lowerATOMIC_LOAD_OP(
-    SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
+    SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG,
+    unsigned Opcode) const {
   auto *Node = cast<AtomicSDNode>(N);
-  unsigned Opcode;
-  switch (Node->getOpcode()) {
-  case ISD::ATOMIC_SWAP:
-    Opcode = SystemZISD::ATOMIC_SWAPW;
-    break;
-  case ISD::ATOMIC_LOAD_ADD:
-    Opcode = SystemZISD::ATOMIC_LOADW_ADD;
-    break;
-  case ISD::ATOMIC_LOAD_SUB:
-    Opcode = SystemZISD::ATOMIC_LOADW_SUB;
-    break;
-  case ISD::ATOMIC_LOAD_AND:
-    Opcode = SystemZISD::ATOMIC_LOADW_AND;
-    break;
-  case ISD::ATOMIC_LOAD_OR:
-    Opcode = SystemZISD::ATOMIC_LOADW_OR;
-    break;
-  case ISD::ATOMIC_LOAD_XOR:
-    Opcode = SystemZISD::ATOMIC_LOADW_XOR;
-    break;
-  case ISD::ATOMIC_LOAD_NAND:
-    Opcode = SystemZISD::ATOMIC_LOADW_NAND;
-    break;
-  case ISD::ATOMIC_LOAD_MIN:
-    Opcode = SystemZISD::ATOMIC_LOADW_MIN;
-    break;
-  case ISD::ATOMIC_LOAD_MAX:
-    Opcode = SystemZISD::ATOMIC_LOADW_MAX;
-    break;
-  case ISD::ATOMIC_LOAD_UMIN:
-    Opcode = SystemZISD::ATOMIC_LOADW_UMIN;
-    break;
-  case ISD::ATOMIC_LOAD_UMAX:
-    Opcode = SystemZISD::ATOMIC_LOADW_UMAX;
-    break;
-  default:
-    llvm_unreachable("Unexpected atomic operation");
-  }
 
   EVT NarrowVT = Node->getMemoryVT();
   EVT WideVT = MVT::i32;
@@ -7636,17 +7599,37 @@ SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
     break;
   }
   case ISD::ATOMIC_SWAP:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_SWAPW);
+    break;
   case ISD::ATOMIC_LOAD_ADD:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_ADD);
+    break;
   case ISD::ATOMIC_LOAD_SUB:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_SUB);
+    break;
   case ISD::ATOMIC_LOAD_AND:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_AND);
+    break;
   case ISD::ATOMIC_LOAD_OR:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_OR);
+    break;
   case ISD::ATOMIC_LOAD_XOR:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_XOR);
+    break;
   case ISD::ATOMIC_LOAD_NAND:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_NAND);
+    break;
   case ISD::ATOMIC_LOAD_MIN:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_MIN);
+    break;
   case ISD::ATOMIC_LOAD_MAX:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_MAX);
+    break;
   case ISD::ATOMIC_LOAD_UMIN:
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
+    break;
   case ISD::ATOMIC_LOAD_UMAX:
-    lowerATOMIC_LOAD_OP(N, Results, DAG);
+    lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
     break;
   default:
     llvm_unreachable("Unexpected node to lower");
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index c5bc2ee13a1f2b..ea71c557947fc1 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -359,7 +359,7 @@ class SystemZTargetLowering : public TargetLowering {
   SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LDST_I128(SDValue Op, SelectionDAG &DAG) const;
   void lowerATOMIC_LOAD_OP(SDNode *N, SmallVectorImpl<SDValue> &Results,
-                           SelectionDAG &DAG) const;
+                           SelectionDAG &DAG, unsigned Opcode) const;
   SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;



More information about the llvm-branch-commits mailing list