[llvm-branch-commits] [llvm] SystemZ: Lower sub-word atomicrmw during type legalization (PR #238532)
Matt Arsenault via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 5 11:46:05 PST 2026
https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/238532
>From c0f525047aebef9685db1bfae7715b365c69b56a Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 3 Nov 2026 13:11:29 -0800
Subject: [PATCH 1/2] SystemZ: Lower sub-word atomicrmw during type
legalization
Produce the ATOMIC_LOADW_* nodes when replacing the illegal i8/i16
results, instead of custom lowering the promoted node later. This
removes the no-op custom type legalization case.
Constants are now any-extended, so one test materializes an immediate
with llill instead of lhi.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
.../Target/SystemZ/SystemZISelLowering.cpp | 114 +++++++++---------
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 4 +-
.../test/CodeGen/SystemZ/atomicrmw-xchg-02.ll | 2 +-
3 files changed, 63 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 87129b03044e7d..6b576fc3058e2d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -310,9 +310,8 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
- // Type legalization will convert 8- and 16-bit atomic operations into
- // forms that operate on i32s (but still keeping the original memory VT).
- // Lower them into full i32 operations.
+ // Lower 8- and 16-bit atomic operations into full i32 operations during
+ // type legalization.
setOperationAction(
{ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD, ISD::ATOMIC_LOAD_SUB,
ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR, ISD::ATOMIC_LOAD_XOR,
@@ -5290,19 +5289,52 @@ static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
}
-// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
-// two into the fullword ATOMIC_LOADW_* operation given by Opcode.
-SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
- SelectionDAG &DAG,
- unsigned Opcode) const {
- auto *Node = cast<AtomicSDNode>(Op.getNode());
+// N is an 8- or 16-bit ATOMIC_LOAD_* operation. Lower it into the fullword
+// ATOMIC_LOADW_* operation.
+void SystemZTargetLowering::lowerATOMIC_LOAD_OP(
+ SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
+ auto *Node = cast<AtomicSDNode>(N);
+ unsigned Opcode;
+ switch (Node->getOpcode()) {
+ case ISD::ATOMIC_SWAP:
+ Opcode = SystemZISD::ATOMIC_SWAPW;
+ break;
+ case ISD::ATOMIC_LOAD_ADD:
+ Opcode = SystemZISD::ATOMIC_LOADW_ADD;
+ break;
+ case ISD::ATOMIC_LOAD_SUB:
+ Opcode = SystemZISD::ATOMIC_LOADW_SUB;
+ break;
+ case ISD::ATOMIC_LOAD_AND:
+ Opcode = SystemZISD::ATOMIC_LOADW_AND;
+ break;
+ case ISD::ATOMIC_LOAD_OR:
+ Opcode = SystemZISD::ATOMIC_LOADW_OR;
+ break;
+ case ISD::ATOMIC_LOAD_XOR:
+ Opcode = SystemZISD::ATOMIC_LOADW_XOR;
+ break;
+ case ISD::ATOMIC_LOAD_NAND:
+ Opcode = SystemZISD::ATOMIC_LOADW_NAND;
+ break;
+ case ISD::ATOMIC_LOAD_MIN:
+ Opcode = SystemZISD::ATOMIC_LOADW_MIN;
+ break;
+ case ISD::ATOMIC_LOAD_MAX:
+ Opcode = SystemZISD::ATOMIC_LOADW_MAX;
+ break;
+ case ISD::ATOMIC_LOAD_UMIN:
+ Opcode = SystemZISD::ATOMIC_LOADW_UMIN;
+ break;
+ case ISD::ATOMIC_LOAD_UMAX:
+ Opcode = SystemZISD::ATOMIC_LOADW_UMAX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic operation");
+ }
- // 32-bit operations need no special handling.
EVT NarrowVT = Node->getMemoryVT();
EVT WideVT = MVT::i32;
- if (NarrowVT == WideVT)
- return Op;
-
int64_t BitSize = NarrowVT.getSizeInBits();
SDValue ChainIn = Node->getChain();
SDValue Addr = Node->getBasePtr();
@@ -5314,8 +5346,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
Opcode = SystemZISD::ATOMIC_LOADW_ADD;
- Src2 = DAG.getSignedConstant(-Const->getSExtValue(), DL,
- Src2.getValueType());
+ Src2 = DAG.getConstant(-Const->getAPIntValue(), DL, Src2.getValueType());
}
SDValue AlignedAddr, BitShift, NegBitShift;
@@ -5326,6 +5357,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
// operations require the source to be shifted in advance. (This shift
// can be folded if the source is constant.) For AND and NAND, the lower
// bits must be set, while for other opcodes they should be left clear.
+ Src2 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Src2);
if (Opcode != SystemZISD::ATOMIC_SWAPW)
Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
DAG.getConstant(32 - BitSize, DL, WideVT));
@@ -5346,32 +5378,25 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
DAG.getConstant(BitSize, DL, WideVT));
SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
-
- SDValue RetOps[2] = { Result, AtomicOp.getValue(1) };
- return DAG.getMergeValues(RetOps, DL);
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Result));
+ Results.push_back(AtomicOp.getValue(1));
}
-// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into
-// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
+// Op is a 32- or 64-bit ATOMIC_LOAD_SUB operation. Convert it into an
+// addition.
SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
SelectionDAG &DAG) const {
auto *Node = cast<AtomicSDNode>(Op.getNode());
EVT MemVT = Node->getMemoryVT();
- if (MemVT == MVT::i32 || MemVT == MVT::i64) {
- // A full-width operation: negate and use LAA(G).
- assert(Op.getValueType() == MemVT && "Mismatched VTs");
- assert(Subtarget.hasInterlockedAccess1() &&
- "Should have been expanded by AtomicExpand pass.");
- SDValue Src2 = Node->getVal();
- SDLoc DL(Src2);
- SDValue NegSrc2 =
+ assert(Op.getValueType() == MemVT && "Mismatched VTs");
+ assert(Subtarget.hasInterlockedAccess1() &&
+ "Should have been expanded by AtomicExpand pass.");
+ SDValue Src2 = Node->getVal();
+ SDLoc DL(Src2);
+ SDValue NegSrc2 =
DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), Src2);
- return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
- Node->getChain(), Node->getBasePtr(), NegSrc2,
- Node->getMemOperand());
- }
-
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB);
+ return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, Node->getChain(),
+ Node->getBasePtr(), NegSrc2, Node->getMemOperand());
}
// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
@@ -7340,32 +7365,12 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
return lowerVECREDUCE_ADD(Op, DAG);
case ISD::ATOMIC_FENCE:
return lowerATOMIC_FENCE(Op, DAG);
- case ISD::ATOMIC_SWAP:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
case ISD::ATOMIC_STORE:
return lowerATOMIC_STORE(Op, DAG);
case ISD::ATOMIC_LOAD:
return lowerATOMIC_LOAD(Op, DAG);
- case ISD::ATOMIC_LOAD_ADD:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
case ISD::ATOMIC_LOAD_SUB:
return lowerATOMIC_LOAD_SUB(Op, DAG);
- case ISD::ATOMIC_LOAD_AND:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND);
- case ISD::ATOMIC_LOAD_OR:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR);
- case ISD::ATOMIC_LOAD_XOR:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR);
- case ISD::ATOMIC_LOAD_NAND:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND);
- case ISD::ATOMIC_LOAD_MIN:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN);
- case ISD::ATOMIC_LOAD_MAX:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX);
- case ISD::ATOMIC_LOAD_UMIN:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
- case ISD::ATOMIC_LOAD_UMAX:
- return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
return lowerATOMIC_CMP_SWAP(Op, DAG);
case ISD::STACKSAVE:
@@ -7641,6 +7646,7 @@ SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
case ISD::ATOMIC_LOAD_MAX:
case ISD::ATOMIC_LOAD_UMIN:
case ISD::ATOMIC_LOAD_UMAX:
+ lowerATOMIC_LOAD_OP(N, Results, DAG);
break;
default:
llvm_unreachable("Unexpected node to lower");
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index d90a8b9c7ff095..c5bc2ee13a1f2b 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -358,8 +358,8 @@ class SystemZTargetLowering : public TargetLowering {
SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_LDST_I128(SDValue Op, SelectionDAG &DAG) const;
- SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
- unsigned Opcode) const;
+ void lowerATOMIC_LOAD_OP(SDNode *N, SmallVectorImpl<SDValue> &Results,
+ SelectionDAG &DAG) const;
SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
index 2cc6b2b9aea8b0..7113f04e32e49e 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
@@ -44,7 +44,7 @@ define i16 @f1(ptr %src, i16 %b) {
; a register and use the sequence above.
define i16 @f2(ptr %src) {
; CHECK-LABEL: f2:
-; CHECK: lhi [[VALUE:%r[0-9]+]], -25536
+; CHECK: llill [[VALUE:%r[0-9]+]], 40000
; CHECK: risbg {{%r[0-9]+}}, [[VALUE]], 32, 47, 16
; CHECK: br %r14
;
>From 1d32fdd842ba1650984ef1fc212c539fc5eb221e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 5 Nov 2026 11:44:28 -0800
Subject: [PATCH 2/2] Pass the opcode into lowerATOMIC_LOAD_OP
---
.../Target/SystemZ/SystemZISelLowering.cpp | 63 +++++++------------
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 2 +-
2 files changed, 24 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 6b576fc3058e2d..ec10c04e723f6f 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -5292,46 +5292,9 @@ static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
// N is an 8- or 16-bit ATOMIC_LOAD_* operation. Lower it into the fullword
// ATOMIC_LOADW_* operation.
void SystemZTargetLowering::lowerATOMIC_LOAD_OP(
- SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
+ SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG,
+ unsigned Opcode) const {
auto *Node = cast<AtomicSDNode>(N);
- unsigned Opcode;
- switch (Node->getOpcode()) {
- case ISD::ATOMIC_SWAP:
- Opcode = SystemZISD::ATOMIC_SWAPW;
- break;
- case ISD::ATOMIC_LOAD_ADD:
- Opcode = SystemZISD::ATOMIC_LOADW_ADD;
- break;
- case ISD::ATOMIC_LOAD_SUB:
- Opcode = SystemZISD::ATOMIC_LOADW_SUB;
- break;
- case ISD::ATOMIC_LOAD_AND:
- Opcode = SystemZISD::ATOMIC_LOADW_AND;
- break;
- case ISD::ATOMIC_LOAD_OR:
- Opcode = SystemZISD::ATOMIC_LOADW_OR;
- break;
- case ISD::ATOMIC_LOAD_XOR:
- Opcode = SystemZISD::ATOMIC_LOADW_XOR;
- break;
- case ISD::ATOMIC_LOAD_NAND:
- Opcode = SystemZISD::ATOMIC_LOADW_NAND;
- break;
- case ISD::ATOMIC_LOAD_MIN:
- Opcode = SystemZISD::ATOMIC_LOADW_MIN;
- break;
- case ISD::ATOMIC_LOAD_MAX:
- Opcode = SystemZISD::ATOMIC_LOADW_MAX;
- break;
- case ISD::ATOMIC_LOAD_UMIN:
- Opcode = SystemZISD::ATOMIC_LOADW_UMIN;
- break;
- case ISD::ATOMIC_LOAD_UMAX:
- Opcode = SystemZISD::ATOMIC_LOADW_UMAX;
- break;
- default:
- llvm_unreachable("Unexpected atomic operation");
- }
EVT NarrowVT = Node->getMemoryVT();
EVT WideVT = MVT::i32;
@@ -7636,17 +7599,37 @@ SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
break;
}
case ISD::ATOMIC_SWAP:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_SWAPW);
+ break;
case ISD::ATOMIC_LOAD_ADD:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_ADD);
+ break;
case ISD::ATOMIC_LOAD_SUB:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_SUB);
+ break;
case ISD::ATOMIC_LOAD_AND:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_AND);
+ break;
case ISD::ATOMIC_LOAD_OR:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_OR);
+ break;
case ISD::ATOMIC_LOAD_XOR:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_XOR);
+ break;
case ISD::ATOMIC_LOAD_NAND:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_NAND);
+ break;
case ISD::ATOMIC_LOAD_MIN:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_MIN);
+ break;
case ISD::ATOMIC_LOAD_MAX:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_MAX);
+ break;
case ISD::ATOMIC_LOAD_UMIN:
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
+ break;
case ISD::ATOMIC_LOAD_UMAX:
- lowerATOMIC_LOAD_OP(N, Results, DAG);
+ lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
break;
default:
llvm_unreachable("Unexpected node to lower");
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index c5bc2ee13a1f2b..ea71c557947fc1 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -359,7 +359,7 @@ class SystemZTargetLowering : public TargetLowering {
SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_LDST_I128(SDValue Op, SelectionDAG &DAG) const;
void lowerATOMIC_LOAD_OP(SDNode *N, SmallVectorImpl<SDValue> &Results,
- SelectionDAG &DAG) const;
+ SelectionDAG &DAG, unsigned Opcode) const;
SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;
More information about the llvm-branch-commits
mailing list