[libcxx-commits] [clang] [clang-tools-extra] [libcxx] [lld] [llvm] [mlir] [lldb][bazel] Add targets for missing plugins (PR #234323)
Lexi Bromfield via libcxx-commits
libcxx-commits at lists.llvm.org
Sat Oct 24 09:39:15 PDT 2026
https://github.com/lexi-nadia updated https://github.com/llvm/llvm-project/pull/234323
>From 0c27c797b755834c6d62734b1ba10bc97fe0453f Mon Sep 17 00:00:00 2001
From: Lexi Bromfield <lexinadia at google.com>
Date: Wed, 21 Oct 2026 18:44:01 +0000
Subject: [PATCH 01/37] [lldb][bazel] Add targets for the remaining
unconditional plugins
CMake builds these plugins on every host, but the Bazel overlay had no
targets for them:
- PluginABILoongArch
- PluginDynamicLoaderFreeBSDKernel
- PluginPlatformAIX
- PluginPlatformEmscripten
- PluginPlatformFreeBSD
- PluginPlatformNetBSD
- PluginPlatformOpenBSD
- PluginTypeSystemClike
I also added these to the DEFAULT_PLUGINS list. I'm less confident that
this is the correct thing to do, so let me know if you think I should
revert it.
For now, I'm still leaving out plugins that CMake only builds with an
optional dependency or on a specific host:
- ProcessFreeBSDKernelCore (FreeBSD hosts, libkvm)
- TraceIntelPT (LLDB_BUILD_INTEL_PT, libipt)
- The TreeSitter highlighters (LLDB_ENABLE_TREESITTER)
Testing:
- `bazel build --config=generic_clang @llvm-project//lldb/...` succeeds.
- `lldb -b -o 'platform list' -o 'plugin list'` shows the new platforms,
the sysv-loongarch ABI, the freebsd-kernel dynamic loader and the cpp
type system.
---
.../lldb/source/Plugins/BUILD.bazel | 139 ++++++++++++++++++
.../lldb/source/Plugins/plugin_config.bzl | 8 +
2 files changed, 147 insertions(+)
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index 1ba0b74cd682e6..3332ee3cbfa0c0 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -158,6 +158,22 @@ alias(
actual = ":PluginsClang",
)
+cc_library(
+ name = "PluginTypeSystemClike",
+ srcs = glob(["TypeSystem/Clike/*.cpp"]),
+ hdrs = glob(["TypeSystem/Clike/*.h"]),
+ includes = [".."],
+ deps = [
+ "//clang:basic",
+ "//lldb:Core",
+ "//lldb:Headers",
+ "//lldb:Symbol",
+ "//lldb:Utility",
+ "//llvm:Support",
+ "//llvm:TargetParser",
+ ],
+)
+
alias(
name = "PluginExpressionParserClang",
actual = ":PluginsClang",
@@ -590,6 +606,91 @@ cc_library(
],
)
+cc_library(
+ name = "PluginPlatformAIX",
+ srcs = glob(["Platform/AIX/*.cpp"]),
+ hdrs = glob(["Platform/AIX/*.h"]),
+ includes = [".."],
+ local_defines = ["_ALL_SOURCE"],
+ deps = [
+ ":PluginPlatformPOSIX",
+ ":PluginsClang",
+ "//lldb:Breakpoint",
+ "//lldb:Core",
+ "//lldb:Headers",
+ "//lldb:Host",
+ "//lldb:Interpreter",
+ "//lldb:Target",
+ "//lldb:Utility",
+ ],
+)
+
+cc_library(
+ name = "PluginPlatformEmscripten",
+ srcs = glob(["Platform/Emscripten/*.cpp"]),
+ hdrs = glob(["Platform/Emscripten/*.h"]),
+ includes = [".."],
+ deps = [
+ ":PluginPlatformPOSIX",
+ "//lldb:Core",
+ "//lldb:Headers",
+ "//lldb:Host",
+ "//lldb:Target",
+ "//lldb:Utility",
+ ],
+)
+
+cc_library(
+ name = "PluginPlatformFreeBSD",
+ srcs = glob(["Platform/FreeBSD/*.cpp"]),
+ hdrs = glob(["Platform/FreeBSD/*.h"]),
+ includes = [".."],
+ deps = [
+ ":PluginPlatformPOSIX",
+ ":PluginsClang",
+ "//lldb:Breakpoint",
+ "//lldb:Core",
+ "//lldb:Headers",
+ "//lldb:Host",
+ "//lldb:Target",
+ "//lldb:Utility",
+ "//llvm:TargetParser",
+ ],
+)
+
+cc_library(
+ name = "PluginPlatformNetBSD",
+ srcs = glob(["Platform/NetBSD/*.cpp"]),
+ hdrs = glob(["Platform/NetBSD/*.h"]),
+ includes = [".."],
+ deps = [
+ ":PluginPlatformPOSIX",
+ ":PluginsClang",
+ "//lldb:Breakpoint",
+ "//lldb:Core",
+ "//lldb:Headers",
+ "//lldb:Host",
+ "//lldb:Target",
+ "//lldb:Utility",
+ ],
+)
+
+cc_library(
+ name = "PluginPlatformOpenBSD",
+ srcs = glob(["Platform/OpenBSD/*.cpp"]),
+ hdrs = glob(["Platform/OpenBSD/*.h"]),
+ includes = [".."],
+ deps = [
+ ":PluginPlatformPOSIX",
+ "//lldb:Breakpoint",
+ "//lldb:Core",
+ "//lldb:Headers",
+ "//lldb:Host",
+ "//lldb:Target",
+ "//lldb:Utility",
+ ],
+)
+
gentbl_cc_library(
name = "PlatformAndroidProperties",
strip_include_prefix = "Platform/Android",
@@ -872,6 +973,24 @@ cc_library(
],
)
+cc_library(
+ name = "PluginABILoongArch",
+ srcs = glob(["ABI/LoongArch/*.cpp"]),
+ hdrs = glob(["ABI/LoongArch/*.h"]),
+ includes = [".."],
+ deps = [
+ ":PluginProcessUtility",
+ "//lldb:Core",
+ "//lldb:Headers",
+ "//lldb:Symbol",
+ "//lldb:Target",
+ "//lldb:Utility",
+ "//llvm:Core",
+ "//llvm:Support",
+ "//llvm:TargetParser",
+ ],
+)
+
cc_library(
name = "PluginABIMips",
srcs = glob(["ABI/Mips/*.cpp"]),
@@ -1109,6 +1228,26 @@ alias(
actual = ":PluginPlatformMacOSX",
)
+cc_library(
+ name = "PluginDynamicLoaderFreeBSDKernel",
+ srcs = glob(["DynamicLoader/FreeBSD-Kernel/*.cpp"]),
+ hdrs = glob(["DynamicLoader/FreeBSD-Kernel/*.h"]),
+ includes = [".."],
+ deps = [
+ ":PluginObjectFileELF",
+ "//lldb:Breakpoint",
+ "//lldb:Core",
+ "//lldb:Headers",
+ "//lldb:Host",
+ "//lldb:Interpreter",
+ "//lldb:Symbol",
+ "//lldb:Target",
+ "//lldb:Utility",
+ "//llvm:BinaryFormat",
+ "//llvm:Support",
+ ],
+)
+
cc_library(
name = "PluginArchitecturePPC64",
srcs = glob(["Architecture/PPC64/*.cpp"]),
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/plugin_config.bzl b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/plugin_config.bzl
index 9a64cb69546e5f..94a086e50aff72 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/plugin_config.bzl
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/plugin_config.bzl
@@ -10,6 +10,7 @@ DEFAULT_PLUGINS = [
"ABIAArch64",
"ABIARM",
"ABIHexagon",
+ "ABILoongArch",
"ABIMips",
"ABIMSP430",
"ABIPowerPC",
@@ -30,6 +31,7 @@ DEFAULT_PLUGINS = [
"HighlighterClang",
"HighlighterDefault",
"DynamicLoaderDarwinKernel",
+ "DynamicLoaderFreeBSDKernel",
"DynamicLoaderHexagonDYLD",
"DynamicLoaderMacOSXDYLD",
"DynamicLoaderPosixDYLD",
@@ -69,10 +71,15 @@ DEFAULT_PLUGINS = [
"ObjectFilePECOFF",
"ObjectFilePlaceholder",
"ObjectFileWasm",
+ "PlatformAIX",
"PlatformAndroid",
+ "PlatformEmscripten",
+ "PlatformFreeBSD",
"PlatformGDB",
"PlatformLinux",
"PlatformMacOSX",
+ "PlatformNetBSD",
+ "PlatformOpenBSD",
"PlatformQemuUser",
"PlatformWindows",
"ProcessElfCore",
@@ -98,6 +105,7 @@ DEFAULT_PLUGINS = [
"SystemRuntimeMacOSX",
"TraceExporterCTF",
"TypeSystemClang",
+ "TypeSystemClike",
"TypeSystemFortran",
"UnwindAssemblyInstEmulation",
"UnwindAssemblyX86",
>From 33bb8249a9edddb8fcbf5002d9a012ffd772dd03 Mon Sep 17 00:00:00 2001
From: Nishant Patel <nishant.b.patel at intel.com>
Date: Fri, 23 Oct 2026 20:46:01 -0700
Subject: [PATCH 02/37] [MLIR][XeGPU] Rematerialize layout conversion sources
after conflict resolution (#230669)
This PR separates layout conflict resolution from rematerialization. The
resolver now inserts xegpu.convert_layout for conflicts ....a following
step clones eligible, cheap vector producer chains in the target layout
and replaces those conversions. This avoids SLM round trips.
---
.../XeGPU/Transforms/XeGPULayoutImpl.h | 19 +-
.../XeGPU/Transforms/XeGPULayoutImpl.cpp | 88 ++++++--
.../XeGPU/Transforms/XeGPUPropagateLayout.cpp | 61 +++---
.../rematerialize-conversion-sources.mlir | 204 ++++++++++++++++++
.../XeGPU/resolve-layout-conflicts.mlir | 56 ++---
.../lib/Dialect/XeGPU/TestXeGPUTransforms.cpp | 32 +++
6 files changed, 375 insertions(+), 85 deletions(-)
create mode 100644 mlir/test/Dialect/XeGPU/rematerialize-conversion-sources.mlir
diff --git a/mlir/include/mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h b/mlir/include/mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h
index 8921380093e108..d587dd2bde7303 100644
--- a/mlir/include/mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h
+++ b/mlir/include/mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h
@@ -18,7 +18,6 @@
#include "llvm/ADT/STLFunctionalExtras.h"
namespace mlir {
-
class VectorType;
class OpOperand;
class OpResult;
@@ -34,13 +33,15 @@ class TensorDescType;
} // namespace xegpu
namespace xegpu {
-
LogicalResult propagateLayouts(OpBuilder &builder, Operation *target,
LayoutKind layoutKind, unsigned indexBitWidth,
bool printOnly = false);
LogicalResult resolveLayoutConflicts(Operation *target);
+/// Replace layout conversions of cheap producer chains by recomputation.
+void rematerializeConversionSources(OpBuilder &builder, Operation *target);
+
/// Sink `xegpu.convert_layout` ops past the elementwise operations they feed,
/// so that an elementwise op runs in the coarser layout.
void sinkElementwiseConversions(OpBuilder &builder, Operation *target);
@@ -465,14 +466,18 @@ completeDpasMxLaneLayoutFromInstData(DistributeLayoutAttr aLayout,
/// users and determine the expected layout accordingly.
DistributeLayoutAttr getConsumerLayoutAt(OpOperand &operand);
-/// Returns true if `op` is safe and cheap to clone: it has no side effects,
-/// no regions, and all of its operands are themselves trivially
-/// rematerializable (e.g. `vector.step`, splat `arith.constant`, or
-/// `vector.create_mask` whose operands are constants).
+/// Returns true if the vector produced by `op` is cheap to recompute in any
+/// layout, i.e. `rematerialize` would succeed: a bounded chain of pure vector
+/// generators (`vector.step`, splat `arith.constant`, `vector.create_mask`,
+/// scalar `vector.broadcast`) and layout-preserving elementwise ops.
bool isTriviallyRematerializable(Operation *op);
+/// Clone a bounded chain of pure vector generators and layout-preserving
+/// elementwise ops without layout attributes. Each producer is cloned once,
+/// immediately after its original; scalar operands remain shared. Return null
+/// if the chain is ineligible, without modifying the IR.
+Value rematerialize(OpBuilder &builder, Value value);
} // namespace xegpu
-
} // namespace mlir
#endif // MLIR_DIALECT_XEGPU_UTILS_XEGPUUTILS_H_
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPULayoutImpl.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPULayoutImpl.cpp
index 649231342e51bd..8e260fc03fdec8 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPULayoutImpl.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPULayoutImpl.cpp
@@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
@@ -21,12 +22,14 @@
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
#include "mlir/IR/Builders.h"
+#include "mlir/IR/IRMapping.h"
#include "mlir/IR/Operation.h"
#include "mlir/IR/ValueRange.h"
#include "mlir/Interfaces/ControlFlowInterfaces.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Transforms/DialectConversion.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/Support/FormatVariadic.h"
#include <cstdint>
@@ -132,25 +135,84 @@ static xegpu::DistributeLayoutAttr getLayoutFromUsePoints(Value result) {
return layout;
}
-// Returns true if `op` is safe and cheap to clone (no side effects, no
-// regions, and all operands are themselves trivially rematerializable, e.g.
-// block-arg-free pure value generators such as `vector.step`, splat
-// `arith.constant`, or `vector.create_mask` whose operands are constants).
-bool xegpu::isTriviallyRematerializable(Operation *op) {
- if (!op || op->getNumRegions() != 0)
- return false;
- if (!isMemoryEffectFree(op))
+static constexpr unsigned kMaxRematerializableChainOps = 16;
+
+/// Accept pure vector generators and elementwise ops that preserve vector
+/// shape. Generators have no vector operands; elementwise ops share their
+/// result layout with their vector operands, so the chain needs no layout
+/// inference.
+static bool isLayoutPreservingVectorProducer(Operation *op) {
+ if (op->getNumResults() != 1 || op->getNumRegions() != 0 ||
+ !isa<VectorType>(op->getResult(0).getType()) || !isMemoryEffectFree(op))
return false;
- for (Value v : op->getOperands()) {
- Operation *defOp = v.getDefiningOp();
- if (!defOp)
- return false;
- if (!isTriviallyRematerializable(defOp))
+ if (auto constant = dyn_cast<arith::ConstantOp>(op)) {
+ auto value = dyn_cast<DenseElementsAttr>(constant.getValue());
+ return value && value.isSplat();
+ }
+ if (isa<vector::StepOp, vector::CreateMaskOp>(op))
+ return true;
+ if (auto broadcast = dyn_cast<vector::BroadcastOp>(op))
+ return !isa<VectorType>(broadcast.getSource().getType());
+ return OpTrait::hasElementwiseMappableTraits(op);
+}
+
+/// Collect each eligible producer once, producers first, for cloning with an
+/// IRMapping. Reject chains exceeding the duplication limit or reaching an
+/// unsupported producer or vector block argument.
+static bool collectRematerializableChain(Operation *root,
+ SmallVectorImpl<Operation *> &chain) {
+ // Re-push each op below its producers to obtain an iterative post-order walk.
+ SmallVector<std::pair<Operation *, bool>> worklist{{root, false}};
+ DenseSet<Operation *> visited;
+ while (!worklist.empty()) {
+ auto [op, expanded] = worklist.pop_back_val();
+ if (expanded) {
+ chain.push_back(op);
+ continue;
+ }
+ if (!visited.insert(op).second)
+ continue;
+ if (visited.size() > kMaxRematerializableChainOps ||
+ !isLayoutPreservingVectorProducer(op))
return false;
+ worklist.emplace_back(op, true);
+ for (Value operand : op->getOperands()) {
+ if (!isa<VectorType>(operand.getType()))
+ continue;
+ Operation *producer = operand.getDefiningOp();
+ if (!producer)
+ return false;
+ if (!visited.contains(producer))
+ worklist.emplace_back(producer, false);
+ }
}
return true;
}
+bool xegpu::isTriviallyRematerializable(Operation *op) {
+ if (!op)
+ return false;
+ SmallVector<Operation *> chain;
+ return collectRematerializableChain(op, chain);
+}
+
+Value xegpu::rematerialize(OpBuilder &builder, Value value) {
+ Operation *producer = value.getDefiningOp();
+ SmallVector<Operation *> chain;
+ if (!producer || !collectRematerializableChain(producer, chain))
+ return nullptr;
+ // Clone producers first so vector operands map to clones; scalars stay
+ // shared.
+ OpBuilder::InsertionGuard guard(builder);
+ IRMapping mapping;
+ for (Operation *op : chain) {
+ builder.setInsertionPointAfter(op);
+ Operation *clone = builder.clone(*op, mapping);
+ removeLayoutAttrs(clone);
+ }
+ return mapping.lookup(value);
+}
+
// For regular operations: First the result layouts are propagated from uses.
// Then the result layouts are propagated to uses (operands).
static void propagateResultsToRegularOperands(Operation *op) {
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
index 3629e122b1bf92..69f927fc0d4943 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
@@ -59,7 +59,6 @@ using namespace mlir;
using namespace mlir::dataflow;
namespace {
-
//===----------------------------------------------------------------------===//
// LayoutInfo
//===----------------------------------------------------------------------===//
@@ -1544,7 +1543,6 @@ void RunLayoutInfoPropagation::printAnalysisResult(llvm::raw_ostream &os) {
}
namespace {
-
//===----------------------------------------------------------------------===//
// ResolveLayoutConflicts
//===----------------------------------------------------------------------===//
@@ -1583,7 +1581,6 @@ struct ResolveLayoutConflicts {
LogicalResult resolveVectorConsumer(OpOperand &operand);
LogicalResult assignResultLayout(OpResult &result);
};
-
} // namespace
LogicalResult ResolveLayoutConflicts::run() {
@@ -1713,26 +1710,6 @@ ResolveLayoutConflicts::resolveVectorConsumer(OpOperand &operand) {
return success();
}
- // If the producer is trivially rematerializable (e.g. `vector.step`, splat
- // `arith.constant`), clone it and stamp the consumer's expected layout on
- // the clone instead of inserting a `xegpu.convert_layout`. The convert
- // would otherwise lower to a cross-subgroup data movement through SLM at
- // WG-to-SG distribution time, which is more expensive than
- // recomputing a pure value generator.
- if (auto *producerOp = vectorValue.getDefiningOp();
- producerOp && producerOp->getNumResults() == 1 &&
- isa<OpResult>(vectorValue) &&
- xegpu::isTriviallyRematerializable(producerOp)) {
- builder.setInsertionPointAfter(producerOp);
- Operation *clone = builder.clone(*producerOp);
- OpResult cloneResult = clone->getResult(0);
- // Drop the inherited producer layout so the new layout takes effect
- xegpu::removeLayoutAttr(cloneResult);
- xegpu::setDistributeLayoutAttr(cloneResult, consumerLayout);
- operand.set(cloneResult);
- return success();
- }
-
// Insert a convert_layout op to resolve the conflict.
builder.setInsertionPointAfterValue(vectorValue);
auto convertOp = xegpu::ConvertLayoutOp::create(
@@ -1835,6 +1812,42 @@ static LogicalResult updateOpWithForwardFill(mlir::OpBuilder &builder,
return success();
}
+/// Replace layout conversions of cheap producer chains by recomputation.
+///
+/// %step = vector.step {L1}
+/// %cast = arith.index_castui %step {L1}
+/// %cvt = xegpu.convert_layout %cast : L1 -> L2
+/// use(%cast) {L1}
+/// use(%cvt) {L2}
+///
+/// becomes
+///
+/// %step = vector.step {L1}
+/// %step' = vector.step
+/// %cast = arith.index_castui %step {L1}
+/// %cast' = arith.index_castui %step'
+/// use(%cast) {L1}
+/// use(%cast') {L2}
+///
+/// Clone the whole vector producer chain so the original and rematerialized
+/// values can serve consumers with different layouts. Clones carry no layout
+/// attributes; layout-dependent passes recover them from the use points. For
+/// small, layout-preserving chains, recomputation can avoid cross-subgroup data
+/// movement through SLM.
+void xegpu::rematerializeConversionSources(OpBuilder &builder,
+ Operation *parentOp) {
+ SmallVector<xegpu::ConvertLayoutOp> conversions;
+ parentOp->walk(
+ [&](xegpu::ConvertLayoutOp convert) { conversions.push_back(convert); });
+ for (xegpu::ConvertLayoutOp convert : conversions) {
+ Value clone = xegpu::rematerialize(builder, convert.getSource());
+ if (!clone)
+ continue;
+ convert.getResult().replaceAllUsesWith(clone);
+ convert.erase();
+ }
+}
+
/// Optimize elementwise operations by sinking costly layout conversion.
///
/// %m = vector.create_mask ... {L1}
@@ -1999,7 +2012,6 @@ struct XeGPUPropagateLayoutPass final
: XeGPUPropagateLayoutBase(std::move(options)) {}
void runOnOperation() override;
};
-
} // namespace
LogicalResult xegpu::propagateLayouts(OpBuilder &builder, Operation *target,
@@ -2108,6 +2120,7 @@ void XeGPUPropagateLayoutPass::runOnOperation() {
signalPassFailure();
return;
}
+ xegpu::rematerializeConversionSources(builder, getOperation());
if (layoutKind == xegpu::LayoutKind::InstData)
xegpu::sinkElementwiseConversions(builder, getOperation());
}
diff --git a/mlir/test/Dialect/XeGPU/rematerialize-conversion-sources.mlir b/mlir/test/Dialect/XeGPU/rematerialize-conversion-sources.mlir
new file mode 100644
index 00000000000000..8d66079652a460
--- /dev/null
+++ b/mlir/test/Dialect/XeGPU/rematerialize-conversion-sources.mlir
@@ -0,0 +1,204 @@
+// RUN: mlir-opt --test-xegpu-rematerialize-conversion-sources -allow-unregistered-dialect -split-input-file %s | FileCheck %s
+
+// Row and column slices require different subgroup distributions.
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+#col = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>
+
+// Rematerialization preserves the original value for its other consumer.
+// CHECK-LABEL: gpu.func @step_conversion
+// CHECK: %[[STEP0:.*]] = vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
+// CHECK-NEXT: %[[STEP1:.*]] = vector.step : vector<32xindex>
+// CHECK: arith.muli %[[STEP1]], %{{.*}} {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>} : vector<32xindex>
+// CHECK: vector.broadcast %[[STEP0]]
+// CHECK-NOT: xegpu.convert_layout
+gpu.module @test {
+gpu.func @step_conversion() kernel {
+ %cst32 = arith.constant {layout_result_0 = #col} dense<32> : vector<32xindex>
+ %step = vector.step {layout_result_0 = #row} : vector<32xindex>
+ %cvt = xegpu.convert_layout %step <{input_layout = #row, target_layout = #col}> : vector<32xindex>
+ %col = arith.muli %cvt, %cst32 {layout_result_0 = #col} : vector<32xindex>
+ %rowb = vector.broadcast %step {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} : vector<32xindex> to vector<32x32xindex>
+ "some_use"(%col, %rowb) : (vector<32xindex>, vector<32x32xindex>) -> ()
+ gpu.return
+}
+}
+
+// -----
+
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+#col = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>
+
+// The cloned elementwise chain shares no vector operands with the original.
+// CHECK-LABEL: gpu.func @elementwise_chain_conversion
+// CHECK-DAG: %[[CST1:.*]] = arith.constant dense<32> : vector<32xindex>
+// CHECK-DAG: %[[STEP1:.*]] = vector.step : vector<32xindex>
+// CHECK: %[[MUL1:.*]] = arith.muli %[[STEP1]], %[[CST1]] : vector<32xindex>
+// CHECK: vector.shape_cast %[[MUL1]]
+// CHECK-NOT: xegpu.convert_layout
+gpu.module @test {
+gpu.func @elementwise_chain_conversion() kernel {
+ %cst32 = arith.constant {layout_result_0 = #row} dense<32> : vector<32xindex>
+ %step = vector.step {layout_result_0 = #row} : vector<32xindex>
+ %scaled = arith.muli %step, %cst32 {layout_result_0 = #row} : vector<32xindex>
+ %cvt = xegpu.convert_layout %scaled <{input_layout = #row, target_layout = #col}> : vector<32xindex>
+ %col2d = vector.shape_cast %cvt {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>} : vector<32xindex> to vector<32x1xindex>
+ %rowb = vector.broadcast %scaled {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} : vector<32xindex> to vector<32x32xindex>
+ "some_use"(%col2d, %rowb) : (vector<32x1xindex>, vector<32x32xindex>) -> ()
+ gpu.return
+}
+}
+
+// -----
+
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+#col = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>
+
+// CHECK-LABEL: gpu.func @cast_chain_conversion
+// CHECK: %[[STEP0:.*]] = vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
+// CHECK-NEXT: %[[STEP1:.*]] = vector.step : vector<32xindex>
+// CHECK-NEXT: arith.index_castui %[[STEP0]] {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex> to vector<32xi32>
+// CHECK-NEXT: arith.index_castui %[[STEP1]] : vector<32xindex> to vector<32xi32>
+// CHECK-NOT: xegpu.convert_layout
+gpu.module @test {
+gpu.func @cast_chain_conversion() kernel {
+ %step = vector.step {layout_result_0 = #row} : vector<32xindex>
+ %cast = arith.index_castui %step {layout_result_0 = #row} : vector<32xindex> to vector<32xi32>
+ %cvt = xegpu.convert_layout %cast <{input_layout = #row, target_layout = #col}> : vector<32xi32>
+ %col2d = vector.shape_cast %cvt {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>} : vector<32xi32> to vector<32x1xi32>
+ %row2d = vector.shape_cast %cast {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} : vector<32xi32> to vector<1x32xi32>
+ "some_use"(%col2d, %row2d) : (vector<32x1xi32>, vector<1x32xi32>) -> ()
+ gpu.return
+}
+}
+
+// -----
+
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+#col = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>
+
+// Shared producers are cloned only once.
+// CHECK-LABEL: gpu.func @shared_operand_cloned_once
+// CHECK-COUNT-2: vector.step
+// CHECK-NOT: vector.step
+// CHECK-NOT: xegpu.convert_layout
+// CHECK: gpu.return
+gpu.module @test {
+gpu.func @shared_operand_cloned_once() kernel {
+ %step = vector.step {layout_result_0 = #row} : vector<32xindex>
+ %sq = arith.muli %step, %step {layout_result_0 = #row} : vector<32xindex>
+ %cvt = xegpu.convert_layout %sq <{input_layout = #row, target_layout = #col}> : vector<32xindex>
+ "some_use"(%cvt) : (vector<32xindex>) -> ()
+ gpu.return
+}
+}
+
+// -----
+
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+#col = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>
+
+// Scalar block arguments remain shared because they carry no layout.
+// CHECK-LABEL: gpu.func @scalar_operand_shared
+// CHECK-SAME: (%[[N:.*]]: index)
+// CHECK: vector.create_mask %[[N]] {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xi1>
+// CHECK-NEXT: %[[MASK1:.*]] = vector.create_mask %[[N]] : vector<32xi1>
+// CHECK: "some_use"(%[[MASK1]])
+// CHECK-NOT: xegpu.convert_layout
+gpu.module @test {
+gpu.func @scalar_operand_shared(%n: index) kernel {
+ %mask = vector.create_mask %n {layout_result_0 = #row} : vector<32xi1>
+ %cvt = xegpu.convert_layout %mask <{input_layout = #row, target_layout = #col}> : vector<32xi1>
+ "some_use"(%cvt) : (vector<32xi1>) -> ()
+ gpu.return
+}
+}
+
+// -----
+
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+#col = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>
+
+// All conversion users share one rematerialized value.
+// CHECK-LABEL: gpu.func @conversion_with_multiple_uses
+// CHECK: %[[STEP1:.*]] = vector.step : vector<32xindex>
+// CHECK: "use_a"(%[[STEP1]])
+// CHECK: "use_b"(%[[STEP1]])
+// CHECK-NOT: xegpu.convert_layout
+gpu.module @test {
+gpu.func @conversion_with_multiple_uses() kernel {
+ %step = vector.step {layout_result_0 = #row} : vector<32xindex>
+ %cvt = xegpu.convert_layout %step <{input_layout = #row, target_layout = #col}> : vector<32xindex>
+ "use_a"(%cvt) : (vector<32xindex>) -> ()
+ "use_b"(%cvt) : (vector<32xindex>) -> ()
+ gpu.return
+}
+}
+
+// -----
+
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+#col = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>
+
+// An ineligible producer prevents rematerialization of the chain.
+// CHECK-LABEL: gpu.func @opaque_source_kept
+// CHECK: xegpu.convert_layout
+gpu.module @test {
+gpu.func @opaque_source_kept() kernel {
+ %v = "some_op"() {layout_result_0 = #row} : () -> vector<32xindex>
+ %step = vector.step {layout_result_0 = #row} : vector<32xindex>
+ %sum = arith.addi %v, %step {layout_result_0 = #row} : vector<32xindex>
+ %cvt = xegpu.convert_layout %sum <{input_layout = #row, target_layout = #col}> : vector<32xindex>
+ "some_use"(%cvt) : (vector<32xindex>) -> ()
+ gpu.return
+}
+}
+
+// -----
+
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+
+// Shape-changing broadcasts cannot share one layout across the chain.
+// CHECK-LABEL: gpu.func @shape_changing_source_kept
+// CHECK: xegpu.convert_layout
+gpu.module @test {
+gpu.func @shape_changing_source_kept() kernel {
+ %step = vector.step {layout_result_0 = #row} : vector<32xindex>
+ %b = vector.broadcast %step {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} : vector<32xindex> to vector<32x32xindex>
+ %cvt = xegpu.convert_layout %b <{input_layout = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, target_layout = #xegpu.layout<sg_layout = [1, 32], sg_data = [32, 1]>}> : vector<32x32xindex>
+ "some_use"(%cvt) : (vector<32x32xindex>) -> ()
+ gpu.return
+}
+}
+
+// -----
+
+#row = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>
+#col = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>
+
+// A chain of 17 ops exceeds the duplication limit.
+// CHECK-LABEL: gpu.func @chain_too_long_kept
+// CHECK: xegpu.convert_layout
+gpu.module @test {
+gpu.func @chain_too_long_kept() kernel {
+ %s = vector.step {layout_result_0 = #row} : vector<32xindex>
+ %a1 = arith.addi %s, %s {layout_result_0 = #row} : vector<32xindex>
+ %a2 = arith.addi %a1, %a1 {layout_result_0 = #row} : vector<32xindex>
+ %a3 = arith.addi %a2, %a2 {layout_result_0 = #row} : vector<32xindex>
+ %a4 = arith.addi %a3, %a3 {layout_result_0 = #row} : vector<32xindex>
+ %a5 = arith.addi %a4, %a4 {layout_result_0 = #row} : vector<32xindex>
+ %a6 = arith.addi %a5, %a5 {layout_result_0 = #row} : vector<32xindex>
+ %a7 = arith.addi %a6, %a6 {layout_result_0 = #row} : vector<32xindex>
+ %a8 = arith.addi %a7, %a7 {layout_result_0 = #row} : vector<32xindex>
+ %a9 = arith.addi %a8, %a8 {layout_result_0 = #row} : vector<32xindex>
+ %a10 = arith.addi %a9, %a9 {layout_result_0 = #row} : vector<32xindex>
+ %a11 = arith.addi %a10, %a10 {layout_result_0 = #row} : vector<32xindex>
+ %a12 = arith.addi %a11, %a11 {layout_result_0 = #row} : vector<32xindex>
+ %a13 = arith.addi %a12, %a12 {layout_result_0 = #row} : vector<32xindex>
+ %a14 = arith.addi %a13, %a13 {layout_result_0 = #row} : vector<32xindex>
+ %a15 = arith.addi %a14, %a14 {layout_result_0 = #row} : vector<32xindex>
+ %a16 = arith.addi %a15, %a15 {layout_result_0 = #row} : vector<32xindex>
+ %cvt = xegpu.convert_layout %a16 <{input_layout = #row, target_layout = #col}> : vector<32xindex>
+ "some_use"(%cvt) : (vector<32xindex>) -> ()
+ gpu.return
+}
+}
diff --git a/mlir/test/Dialect/XeGPU/resolve-layout-conflicts.mlir b/mlir/test/Dialect/XeGPU/resolve-layout-conflicts.mlir
index d822b487e53880..e78b854bd0a749 100644
--- a/mlir/test/Dialect/XeGPU/resolve-layout-conflicts.mlir
+++ b/mlir/test/Dialect/XeGPU/resolve-layout-conflicts.mlir
@@ -154,18 +154,18 @@ func.func @bitcast_source_conflict() {
return
}
-// The accumulator `arith.constant` is trivially rematerializable, so the
-// conflict resolver clones it with the consumer's expected layout instead
-// of inserting an `xegpu.convert_layout`
+// A conflicting constant accumulator requires a layout conversion.
// CHECK-LABEL: func.func @multireduction_source_conflict
// CHECK-DAG: %[[V0:.*]] = "some_op"() {layout_result_0 = #xegpu.layout<inst_data = [32, 16]>} : () -> vector<32x32xf16>
// CHECK-DAG: %[[CVT0:.*]] = xegpu.convert_layout %[[V0]]
// CHECK-SAME: <{input_layout = #xegpu.layout<inst_data = [32, 16]>, target_layout = #xegpu.layout<inst_data = [16, 16]>}>
// CHECK-SAME: : vector<32x32xf16>
-// CHECK-DAG: %[[CST:.*]] = arith.constant {layout_result_0 = #xegpu.slice<#xegpu.layout<inst_data = [16, 16]>, dims = [0]>}
+// CHECK-DAG: %[[CST:.*]] = arith.constant {layout_result_0 = #xegpu.layout<inst_data = [32]>}
// CHECK-SAME: dense<0.000000e+00> : vector<32xf16>
-// CHECK-NOT: xegpu.convert_layout %{{.*}} : vector<32xf16>
-// CHECK: %[[MR:.*]] = vector.multi_reduction <add>, %[[CVT0]], %[[CST]]
+// CHECK-DAG: %[[CVT1:.*]] = xegpu.convert_layout %[[CST]]
+// CHECK-SAME: <{input_layout = #xegpu.layout<inst_data = [32]>, target_layout = #xegpu.slice<#xegpu.layout<inst_data = [16, 16]>, dims = [0]>}>
+// CHECK-SAME: : vector<32xf16>
+// CHECK: %[[MR:.*]] = vector.multi_reduction <add>, %[[CVT0]], %[[CVT1]]
// CHECK-SAME: {layout_result_0 = #xegpu.slice<#xegpu.layout<inst_data = [16, 16]>, dims = [0]>}
// CHECK-SAME: [0] : vector<32x32xf16> to vector<32xf16>
// CHECK: return
@@ -356,16 +356,15 @@ func.func @convert_layout() {
return
}
-// A `vector.step` reaches two consumers that need different slice layouts of
-// the anchor's sg_layout (one along dim 0, one along dim 1). The conflict is
-// trivially rematerializable, so the resolver clones the step with the second
-// layout instead of inserting an `xegpu.convert_layout` (which would lower to
-// a cross-subgroup SLM round-trip at WG-to-SG distribution time).
-// CHECK-LABEL: gpu.func @step_clone_via_anchor
-// CHECK-DAG: vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
-// CHECK-DAG: vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>} : vector<32xindex>
-// CHECK-NOT: xegpu.convert_layout {{.*}} : vector<32xindex>
-gpu.func @step_clone_via_anchor(%arg0: i64, %arg1: memref<32x32xf32>) kernel {
+// Conflicting row and column uses of a step require a layout conversion.
+// CHECK-LABEL: gpu.func @step_conflict_via_anchor
+// CHECK: %[[STEP:.*]] = vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
+// CHECK-NEXT: %[[CVT:.*]] = xegpu.convert_layout %[[STEP]]
+// CHECK-SAME: <{input_layout = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>, target_layout = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>}>
+// CHECK-SAME: : vector<32xindex>
+// CHECK-NEXT: arith.muli %[[CVT]], %{{.*}} {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>} : vector<32xindex>
+// CHECK: vector.broadcast %[[STEP]]
+gpu.func @step_conflict_via_anchor(%arg0: i64, %arg1: memref<32x32xf32>) kernel {
%mask = arith.constant {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} dense<true> : vector<32x32xi1>
%cst32 = arith.constant {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>} dense<32> : vector<32xindex>
%step = vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
@@ -381,31 +380,6 @@ gpu.func @step_clone_via_anchor(%arg0: i64, %arg1: memref<32x32xf32>) kernel {
gpu.return
}
-// Test that when a derived value (arith.muli) has a layout conflict on its
-// uses, and the derived value's producer chain is trivially rematerializable
-// (because step and constant stride are themselves trivially rematerializable),
-// the resolver clones the arith.muli instead of inserting an xegpu.convert_layout.
-// CHECK-LABEL: gpu.func @step_muli_clone_via_anchor
-// CHECK-DAG: vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
-// CHECK-DAG: arith.muli {{.*}} {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
-// CHECK-DAG: arith.muli {{.*}} {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>, dims = [1]>} : vector<32xindex>
-// CHECK-NOT: xegpu.convert_layout {{.*}} : vector<32xindex>
-gpu.func @step_muli_clone_via_anchor(%arg0: i64, %arg1: memref<32x32xf32>) kernel {
- %mask = arith.constant {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} dense<true> : vector<32x32xi1>
- %cst32 = arith.constant {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} dense<32> : vector<32xindex>
- %step = vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
- %scaled = arith.muli %step, %cst32 {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>, dims = [0]>} : vector<32xindex>
- %col2d = vector.shape_cast %scaled {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 1]>} : vector<32xindex> to vector<32x1xindex>
- %colb = vector.broadcast %col2d {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} : vector<32x1xindex> to vector<32x32xindex>
- %rowb = vector.broadcast %scaled {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} : vector<32xindex> to vector<32x32xindex>
- %off = arith.addi %colb, %rowb {layout_result_0 = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>} : vector<32x32xindex>
- %v = xegpu.load %arg0[%off], %mask <{layout = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>}>
- : i64, vector<32x32xindex>, vector<32x32xi1> -> vector<32x32xf32>
- %tdesc = xegpu.create_nd_tdesc %arg1 : memref<32x32xf32> -> !xegpu.tensor_desc<32x32xf32>
- xegpu.store_nd %v, %tdesc[0, 0] <{layout = #xegpu.layout<sg_layout = [32, 1], sg_data = [1, 32]>}> : vector<32x32xf32>, !xegpu.tensor_desc<32x32xf32>
- gpu.return
-}
-
// CHECK-LABEL: func.func @extract_source_conflict_with_order
// CHECK-DAG: %[[V0:.*]] = "some_op"() {layout_result_0 = #xegpu.layout<lane_layout = [1, 1, 1, 16], lane_data = [1, 1, 1, 1], order = [2, 3, 0, 1]>} : () -> vector<2x4x16x32xf16>
// CHECK-DAG: %[[CVT:.*]] = xegpu.convert_layout %[[V0]]
diff --git a/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp b/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
index aacfd1a563d972..61b666a4b1a313 100644
--- a/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
+++ b/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
@@ -361,6 +361,37 @@ struct TestXeGPUResolveLayoutConflicts
}
};
+struct TestXeGPURematerializeConversionSources
+ : public PassWrapper<TestXeGPURematerializeConversionSources,
+ OperationPass<gpu::GPUModuleOp>> {
+ MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
+ TestXeGPURematerializeConversionSources)
+
+ StringRef getArgument() const final {
+ return "test-xegpu-rematerialize-conversion-sources";
+ }
+
+ StringRef getDescription() const final {
+ return "Test replacing XeGPU layout conversions of trivially "
+ "rematerializable values by recomputation.";
+ }
+
+ void getDependentDialects(::mlir::DialectRegistry ®istry) const override {
+ registry.insert<xegpu::XeGPUDialect>();
+ registry.insert<gpu::GPUDialect>();
+ registry.insert<vector::VectorDialect>();
+ }
+
+ TestXeGPURematerializeConversionSources() = default;
+ TestXeGPURematerializeConversionSources(
+ const TestXeGPURematerializeConversionSources &pass) = default;
+
+ void runOnOperation() override {
+ OpBuilder builder(&getContext());
+ xegpu::rematerializeConversionSources(builder, getOperation());
+ }
+};
+
struct TestXeGPUSinkElementwiseConversions
: public PassWrapper<TestXeGPUSinkElementwiseConversions,
OperationPass<gpu::GPUModuleOp>> {
@@ -588,6 +619,7 @@ void registerTestXeGPULowerings() {
PassRegistration<TestXeGPUSgToLaneDistribute>();
PassRegistration<TestXeGPUPropagateLayouts>();
PassRegistration<TestXeGPUResolveLayoutConflicts>();
+ PassRegistration<TestXeGPURematerializeConversionSources>();
PassRegistration<TestXeGPUSinkElementwiseConversions>();
PassRegistration<TestXeGPUArrayLengthOptimization>();
PassRegistration<TestXeGPUCoalesceGatherScatter>();
>From 231c432591e66aee3bd6a2e0271a6fae17ab0a73 Mon Sep 17 00:00:00 2001
From: Akash Manna <akash.manna.mymail at gmail.com>
Date: Sat, 24 Oct 2026 09:26:27 +0530
Subject: [PATCH 03/37] [clang][Sema] Expand `#embed` in braced lists that
initialize non-aggregates (#231938)
Fixes #222796
A multi-byte `#embed` inside braces is one `EmbedExpr`, and only
aggregate initialization slices it. Every other consumer of the braced
list counted it as a single element, so `X x = { #embed ... }` called
the constructor with the first byte only, and a reference or fixed enum
silently took the first byte instead of being rejected.
`Sema::getInitsWithEmbedExpanded()` hands those consumers the list with
each multi-element `EmbedExpr` split into one single-element `EmbedExpr`
per byte, the same slices aggregate initialization already builds.
Constructor selection, braced-argument overload resolution and CTAD use
it; the single-element and excess-element checks go through the new
`InitListExpr::hasZeroInits()` / `hasOneInit()`. Aggregates still keep
the whole `EmbedExpr`.
For more reference, see https://github.com/llvm/llvm-project/pull/218262
LLM tools were used for this contribution. I've reviewed, built, and
tested the change myself before pushing to GitHub.
---
clang/docs/ReleaseNotes.md | 3 +
clang/include/clang/AST/Expr.h | 11 ++
clang/include/clang/Sema/Sema.h | 5 +
clang/lib/Sema/SemaInit.cpp | 82 +++++++---
clang/lib/Sema/SemaOverload.cpp | 26 ++--
clang/test/Preprocessor/embed_codegen.cpp | 11 ++
clang/test/Preprocessor/embed_constexpr.c | 4 +
clang/test/Preprocessor/embed_constexpr.cpp | 27 ++++
clang/test/Sema/excess-embed-data.c | 6 +
clang/test/SemaCXX/embed-init-list.cpp | 158 +++++++++++++++++++-
10 files changed, 300 insertions(+), 33 deletions(-)
diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index adcdd911beb09c..44f30666457f94 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -916,6 +916,9 @@ documentation](CIR/index.md) for more information regarding ClangIR's status and
of an array new-expression, or of an array whose elements are of class type.
(#GH128985)
+- Fixed list-initialization from `#embed` passing only the first embedded
+ element to a constructor, reference, scalar or `auto` deduction. (#GH222796)
+
- Fixed a bug where top-level CV qualifiers (such as ``const``) were dropped from pointers modified by Microsoft pointer attributes (like ``__ptr32`` and ``__ptr64``) and WebAssembly's ``__funcref``.
- Fixed a bug where we accepted ``__super`` being qualified by a scope specifier, causing codegen to assertion fail elsewhere. (#GH212988)
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 475feec1a3891b..d77efd9bea6058 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -5419,6 +5419,17 @@ class InitListExpr : public Expr {
return Sum;
}
+ bool hasZeroInits() const { return InitExprs.empty(); }
+
+ /// Whether getNumInitsWithEmbedExpanded() is 1, without walking the list.
+ bool hasOneInit() const {
+ if (InitExprs.size() != 1)
+ return false;
+ auto *EE =
+ dyn_cast<EmbedExpr>(cast<Expr>(InitExprs[0])->IgnoreParenImpCasts());
+ return !EE || EE->getDataElementCount() == 1;
+ }
+
/// Retrieve the set of initializers.
Expr **getInits() { return reinterpret_cast<Expr **>(InitExprs.data()); }
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 9082042c48502d..cd2227f2aa5c69 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -9188,6 +9188,11 @@ class Sema final : public SemaBase {
bool IsStringInit(Expr *Init, const ArrayType *AT);
+ /// The initializers of \p ILE with each multi-element EmbedExpr split into
+ /// one EmbedExpr per embedded element; \p Storage is used only if needed.
+ MultiExprArg getInitsWithEmbedExpanded(InitListExpr *ILE,
+ SmallVectorImpl<Expr *> &Storage);
+
/// Determine whether we can perform aggregate initialization for the purposes
/// of overload resolution.
bool CanPerformAggregateInitializationForOverloadResolution(
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index b55ad09c099a63..fa4ef1ce57f172 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -1718,7 +1718,7 @@ void InitListChecker::CheckComplexType(const InitializedEntity &Entity,
// the element type of the complex type. The first element initializes
// the real part, and the second element intitializes the imaginary part.
- if (IList->getNumInits() < 2)
+ if (IList->hasZeroInits() || IList->hasOneInit())
return CheckScalarType(Entity, IList, DeclType, Index, StructuredList,
StructuredIndex);
@@ -3637,6 +3637,32 @@ bool Sema::CanPerformAggregateInitializationForOverloadResolution(
return !Check.HadError();
}
+MultiExprArg Sema::getInitsWithEmbedExpanded(InitListExpr *ILE,
+ SmallVectorImpl<Expr *> &Storage) {
+ auto AsMultiElementEmbed = [](Expr *Init) {
+ auto *Embed = dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts());
+ return Embed && Embed->getDataElementCount() > 1 ? Embed : nullptr;
+ };
+ if (llvm::none_of(ILE->inits(), AsMultiElementEmbed))
+ return MultiExprArg(ILE->getInits(), ILE->getNumInits());
+
+ Storage.clear();
+ Storage.reserve(ILE->getNumInitsWithEmbedExpanded());
+ for (Expr *Init : ILE->inits()) {
+ EmbedExpr *Embed = AsMultiElementEmbed(Init);
+ if (!Embed) {
+ Storage.push_back(Init);
+ continue;
+ }
+ for (unsigned I = Embed->getStartingElementPos(),
+ E = I + Embed->getDataElementCount();
+ I != E; ++I)
+ Storage.push_back(new (Context) EmbedExpr(Context, Embed->getLocation(),
+ Embed->getData(), I, 1));
+ }
+ return Storage;
+}
+
/// Check that the given Index expression is a valid array designator
/// value. This is essentially just a wrapper around
/// VerifyIntegerConstantExpression that also checks for negative values
@@ -4600,8 +4626,9 @@ static void TryConstructorInitialization(Sema &S,
"argument.");
InitListExpr *ILE =
(IsListInit || IsInitListCopy) ? cast<InitListExpr>(Args[0]) : nullptr;
+ SmallVector<Expr *, 8> ExpandedInits;
MultiExprArg UnwrappedArgs =
- ILE ? MultiExprArg(ILE->getInits(), ILE->getNumInits()) : Args;
+ ILE ? S.getInitsWithEmbedExpanded(ILE, ExpandedInits) : Args;
// The type we're constructing needs to be complete.
if (!S.isCompleteType(Kind.getLocation(), DestType)) {
@@ -4948,7 +4975,7 @@ static void TryReferenceListInitialization(Sema &S,
// (possibly creating temporaries).
// Otherwise, initialize a temporary with the initializer list and
// bind to that.
- if (InitList->getNumInits() == 1) {
+ if (InitList->hasOneInit()) {
Expr *Initializer = InitList->getInit(0);
QualType cv2T2 = S.getCompletedType(Initializer);
Qualifiers T2Quals;
@@ -5042,7 +5069,8 @@ static void TryListInitialization(Sema &S,
// C++ doesn't allow scalar initialization with more than one argument.
// But C99 complex numbers are scalars and it makes sense there.
if (S.getLangOpts().CPlusPlus && DestType->isScalarType() &&
- !DestType->isAnyComplexType() && InitList->getNumInits() > 1) {
+ !DestType->isAnyComplexType() && !InitList->hasZeroInits() &&
+ !InitList->hasOneInit()) {
Sequence.SetFailed(InitializationSequence::FK_TooManyInitsForScalar);
return;
}
@@ -5085,7 +5113,7 @@ static void TryListInitialization(Sema &S,
// (8.5.2 [dcl.init.string]), initialization is performed as described
// in that section.
// - Otherwise, if T is an aggregate, [...] (continue below).
- if (S.getLangOpts().CPlusPlus11 && InitList->getNumInits() == 1 &&
+ if (S.getLangOpts().CPlusPlus11 && InitList->hasOneInit() &&
!IsDesignatedInit) {
if (DestType->isRecordType() && DestType->isAggregateType()) {
QualType InitType = InitList->getInit(0)->getType();
@@ -5167,7 +5195,7 @@ static void TryListInitialization(Sema &S,
// - Otherwise, if the initializer list has no elements and T is a
// class type with a default constructor, the object is
// value-initialized.
- if (InitList->getNumInits() == 0) {
+ if (InitList->hasZeroInits()) {
CXXRecordDecl *RD = DestType->castAsCXXRecordDecl();
if (S.LookupDefaultConstructor(RD)) {
TryValueInitialization(S, Entity, Kind, Sequence, InitList);
@@ -5191,7 +5219,7 @@ static void TryListInitialization(Sema &S,
}
if (S.getLangOpts().CPlusPlus && !DestType->isAggregateType() &&
- InitList->getNumInits() == 1) {
+ InitList->hasOneInit()) {
Expr *E = InitList->getInit(0);
// - Otherwise, if T is an enumeration with a fixed underlying type,
@@ -8465,7 +8493,8 @@ ExprResult InitializationSequence::Perform(Sema &S,
InitListExpr *InitList = cast<InitListExpr>(Args[0]);
S.Diag(InitList->getExprLoc(), diag::warn_cxx98_compat_ctor_list_init)
<< InitList->getSourceRange();
- MultiExprArg Arg(InitList->getInits(), InitList->getNumInits());
+ SmallVector<Expr *, 8> ExpandedInits;
+ MultiExprArg Arg = S.getInitsWithEmbedExpanded(InitList, ExpandedInits);
CurInit = PerformConstructorInitialization(S, UseTemporary ? TempEntity :
Entity,
Kind, Arg, *Step,
@@ -9000,7 +9029,7 @@ static void diagnoseListInit(Sema &S, const InitializedEntity &Entity,
QualType ArrayType = S.Context.getConstantArrayType(
E.withConst(),
llvm::APInt(S.Context.getTypeSize(S.Context.getSizeType()),
- InitList->getNumInits()),
+ InitList->getNumInitsWithEmbedExpanded()),
nullptr, clang::ArraySizeModifier::Normal, 0);
InitializedEntity HiddenArray =
InitializedEntity::InitializeTemporary(ArrayType);
@@ -9041,7 +9070,7 @@ bool InitializationSequence::Diagnose(Sema &S,
Expr *OnlyArg;
if (Args.size() == 1) {
auto *List = dyn_cast<InitListExpr>(Args[0]);
- if (List && List->getNumInits() == 1)
+ if (List && List->hasOneInit())
OnlyArg = List->getInit(0);
else
OnlyArg = Args[0];
@@ -9361,11 +9390,12 @@ bool InitializationSequence::Diagnose(Sema &S,
ArgsRange =
SourceRange(Args.front()->getBeginLoc(), Args.back()->getEndLoc());
+ SmallVector<Expr *, 8> ExpandedInits;
if (Failure == FK_ListConstructorOverloadFailed) {
assert(Args.size() == 1 &&
"List construction from other than 1 argument.");
InitListExpr *InitList = cast<InitListExpr>(Args[0]);
- Args = MultiExprArg(InitList->getInits(), InitList->getNumInits());
+ Args = S.getInitsWithEmbedExpanded(InitList, ExpandedInits);
}
// FIXME: Using "DestType" for the entity we're printing is probably
@@ -9920,6 +9950,20 @@ void InitializationSequence::dump() const {
dump(llvm::errs());
}
+static NarrowingKind getNarrowingKindWithEmbedExpanded(
+ Sema &S, const StandardConversionSequence &SCS, const Expr *Init,
+ APValue &ConstantValue, QualType &ConstantType) {
+ const auto *Embed = dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts());
+ if (!Embed || Embed->getDataElementCount() <= 1)
+ return SCS.getNarrowingKind(S.Context, Init, ConstantValue, ConstantType);
+ for (const IntegerLiteral *Elt : Embed->underlying_data_elements())
+ if (NarrowingKind NK =
+ SCS.getNarrowingKind(S.Context, Elt, ConstantValue, ConstantType);
+ NK != NK_Not_Narrowing)
+ return NK;
+ return NK_Not_Narrowing;
+}
+
static void DiagnoseNarrowingInInitList(Sema &S,
const ImplicitConversionSequence &ICS,
QualType PreNarrowingType,
@@ -9956,8 +10000,8 @@ static void DiagnoseNarrowingInInitList(Sema &S,
// C++11 [dcl.init.list]p7: Check whether this is a narrowing conversion.
APValue ConstantValue;
QualType ConstantType;
- switch (SCS->getNarrowingKind(S.Context, PostInit, ConstantValue,
- ConstantType)) {
+ switch (getNarrowingKindWithEmbedExpanded(S, *SCS, PostInit, ConstantValue,
+ ConstantType)) {
case NK_Not_Narrowing:
case NK_Dependent_Narrowing:
// No narrowing occurred.
@@ -10040,9 +10084,8 @@ static void CheckC23ConstexprInitConversion(Sema &S, QualType FromType,
APValue Value;
QualType PreNarrowingType;
// Reuse C++ narrowing check.
- switch (ICS.Standard.getNarrowingKind(
- S.Context, Init, Value, PreNarrowingType,
- /*IgnoreFloatToIntegralConversion*/ false)) {
+ switch (getNarrowingKindWithEmbedExpanded(S, ICS.Standard, Init, Value,
+ PreNarrowingType)) {
// The value doesn't fit.
case NK_Constant_Narrowing:
S.Diag(Init->getBeginLoc(), diag::err_c23_constexpr_init_not_representable)
@@ -10265,6 +10308,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
(Inits.size() == 1 && Kind.getKind() != InitializationKind::IK_Direct)
? dyn_cast<InitListExpr>(Inits[0])
: nullptr;
+ SmallVector<Expr *, 8> ExpandedInits;
// C++1z [over.match.class.deduct]p1:
// Initialization and overload resolution are performed as described in
@@ -10458,7 +10502,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
// Try list constructors unless the list is empty and the class has one or
// more default constructors, in which case those constructors win.
- if (!ListInit->getNumInits()) {
+ if (ListInit->hasZeroInits()) {
for (NamedDecl *D : Guides) {
auto *FD = dyn_cast<FunctionDecl>(D->getUnderlyingDecl());
if (FD && FD->getMinRequiredArguments() == 0) {
@@ -10466,7 +10510,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
break;
}
}
- } else if (ListInit->getNumInits() == 1) {
+ } else if (ListInit->hasOneInit()) {
// C++ [over.match.class.deduct]:
// As an exception, the first phase in [over.match.list] (considering
// initializer-list constructors) is omitted if the initializer list
@@ -10484,7 +10528,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
Result = TryToResolveOverload(/*OnlyListConstructor*/true);
// Then unwrap the initializer list and try again considering all
// constructors.
- Inits = MultiExprArg(ListInit->getInits(), ListInit->getNumInits());
+ Inits = getInitsWithEmbedExpanded(ListInit, ExpandedInits);
}
// If list-initialization fails, or if we're doing any other kind of
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 72dcb96f098c89..c56f8127834a51 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -4244,6 +4244,7 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
Expr **Args = &From;
unsigned NumArgs = 1;
bool ListInitializing = false;
+ SmallVector<Expr *, 8> ExpandedInits;
if (InitListExpr *InitList = dyn_cast<InitListExpr>(From)) {
// But first, see if there is an init-list-constructor that will work.
OverloadingResult Result = IsInitializerListConstructorConversion(
@@ -4257,8 +4258,10 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
// If we're list-initializing, we pass the individual elements as
// arguments, not the entire list.
- Args = InitList->getInits();
- NumArgs = InitList->getNumInits();
+ MultiExprArg Inits =
+ S.getInitsWithEmbedExpanded(InitList, ExpandedInits);
+ Args = Inits.data();
+ NumArgs = Inits.size();
ListInitializing = true;
}
@@ -5866,7 +5869,7 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
// and the initializer list has a single element that is an
// appropriately-typed string literal (8.5.2 [dcl.init.string]), the
// implicit conversion sequence is the identity conversion.
- if (From->getNumInits() == 1 && !IsDesignatedInit) {
+ if (From->hasOneInit() && !IsDesignatedInit) {
if (ToType->isRecordType() && ToType->isAggregateType()) {
QualType InitType = From->getInit(0)->getType();
if (S.Context.hasSameUnqualifiedType(InitType, ToType) ||
@@ -5905,7 +5908,7 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
// can be implicitly converted to X, the implicit conversion sequence is
// the worst conversion necessary to convert an element of the list to X.
if ((AT || S.isStdInitializerList(ToType, &InitTy)) && !IsDesignatedInit) {
- unsigned e = From->getNumInits();
+ unsigned e = From->getNumInitsWithEmbedExpanded();
ImplicitConversionSequence DfltElt;
DfltElt.setBad(BadConversionSequence::no_conversion, QualType(),
QualType());
@@ -5957,8 +5960,9 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
Result.Standard.setAsIdentityConversion();
Result.Standard.setFromType(InitTy);
Result.Standard.setAllToTypes(InitTy);
- for (unsigned i = 0; i < e; ++i) {
- Expr *Init = From->getInit(i);
+ // An EmbedExpr is a single int prvalue, so one conversion sequence covers
+ // all of its elements; narrowing is checked per element after resolution.
+ for (Expr *Init : From->inits()) {
ImplicitConversionSequence ICS = TryCopyInitialization(
S, Init, InitTy, SuppressUserConversions, InOverloadResolution,
AllowObjCWritebackConversion);
@@ -6046,7 +6050,7 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
// If the initializer list has a single element that is reference-related
// to the parameter type, we initialize the reference from that.
- if (From->getNumInits() == 1 && !IsDesignatedInit) {
+ if (From->hasOneInit() && !IsDesignatedInit) {
Expr *Init = From->getInit(0);
QualType T2 = Init->getType();
@@ -6108,11 +6112,7 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
// - if the initializer list has one element that is not itself an
// initializer list, the implicit conversion sequence is the one
// required to convert the element to the parameter type.
- // Bail out on EmbedExpr as well since we never create EmbedExpr for a
- // single integer.
- unsigned NumInits = From->getNumInits();
- if (NumInits == 1 && !isa<InitListExpr>(From->getInit(0)) &&
- !isa<EmbedExpr>(From->getInit(0))) {
+ if (From->hasOneInit() && !isa<InitListExpr>(From->getInit(0))) {
Result = TryCopyInitialization(
S, From->getInit(0), ToType, SuppressUserConversions,
InOverloadResolution, AllowObjCWritebackConversion);
@@ -6121,7 +6121,7 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
}
// - if the initializer list has no elements, the implicit conversion
// sequence is the identity conversion.
- else if (NumInits == 0) {
+ else if (From->hasZeroInits()) {
Result.setStandard();
Result.Standard.setAsIdentityConversion();
Result.Standard.setFromType(ToType);
diff --git a/clang/test/Preprocessor/embed_codegen.cpp b/clang/test/Preprocessor/embed_codegen.cpp
index 4f2b5f263985a3..9b58131e0a9bfd 100644
--- a/clang/test/Preprocessor/embed_codegen.cpp
+++ b/clang/test/Preprocessor/embed_codegen.cpp
@@ -91,3 +91,14 @@ constexpr struct Wrapper W = {
};
}
+
+struct Ctor {
+ Ctor(int, int);
+};
+
+// CHECK: call void @_ZN4CtorC1Eii(ptr {{.*}}%c, i32 noundef 106, i32 noundef 107)
+void ctor_list() {
+ Ctor c = {
+#embed <jk.txt>
+ };
+}
diff --git a/clang/test/Preprocessor/embed_constexpr.c b/clang/test/Preprocessor/embed_constexpr.c
index e4c85cce459e83..3e36bc74db003d 100644
--- a/clang/test/Preprocessor/embed_constexpr.c
+++ b/clang/test/Preprocessor/embed_constexpr.c
@@ -24,3 +24,7 @@ static constexpr int data4[] = {
static constexpr float data5[] = {
#embed "big_char.txt" suffix(, -1)
};
+
+static constexpr _BitInt(7) data6[] = {
+#embed __FILE__ limit(4) // expected-error {{constexpr initializer evaluates to 82 which is not exactly representable in type 'const _BitInt(7)'}}
+};
diff --git a/clang/test/Preprocessor/embed_constexpr.cpp b/clang/test/Preprocessor/embed_constexpr.cpp
index c51c02def7dfb5..63369599cfbb8e 100644
--- a/clang/test/Preprocessor/embed_constexpr.cpp
+++ b/clang/test/Preprocessor/embed_constexpr.cpp
@@ -1,5 +1,6 @@
// RUN: %clang_cc1 %s -fsyntax-only --embed-dir=%S/Inputs -verify -Wno-c23-extensions
// RUN: %clang_cc1 %s -fsyntax-only --embed-dir=%S/Inputs -verify -fexperimental-new-constant-interpreter -Wno-c23-extensions
+// RUN: %clang_cc1 %s -ast-dump -ast-dump-filter=ctor_list --embed-dir=%S/Inputs -Wno-c23-extensions | FileCheck %s
// expected-no-diagnostics
constexpr int value(int a, int b) {
@@ -104,3 +105,29 @@ constexpr struct Wrapper W = {
};
static_assert(W.d.h[2] == '3');
+
+struct X {
+ template <typename... T>
+ constexpr X(T... t) : sum((0 + ... + int(t))), size(sizeof...(T)) {}
+ int sum;
+ int size;
+};
+
+// CHECK: CXXConstructExpr {{.*}} 'const X' 'void (int, int)' list
+// CHECK-NEXT: EmbedExpr {{.*}} 'int'
+// CHECK-NEXT: begin: 0
+// CHECK-NEXT: number of elements: 1
+// CHECK-NEXT: EmbedExpr {{.*}} 'int'
+// CHECK-NEXT: begin: 1
+// CHECK-NEXT: number of elements: 1
+constexpr X ctor_list = {
+#embed <jk.txt>
+};
+static_assert(ctor_list.size == 2);
+static_assert(ctor_list.sum == ExpectedValue);
+
+constexpr X ctor_paren(
+#embed <jk.txt>
+);
+static_assert(ctor_paren.size == 2);
+static_assert(ctor_paren.sum == ExpectedValue);
diff --git a/clang/test/Sema/excess-embed-data.c b/clang/test/Sema/excess-embed-data.c
index d5b84921abc894..f6d06bff6a7ca5 100644
--- a/clang/test/Sema/excess-embed-data.c
+++ b/clang/test/Sema/excess-embed-data.c
@@ -47,4 +47,10 @@ struct S1 s1 = {
// cpp-error at -2{{excess elements in struct initializer}}
, x
};
+
+int f = {
+#embed __FILE__
+ // c-warning at -1{{excess elements in scalar initializer}}
+ // cpp-error at -3{{excess elements in scalar initializer}}
+};
}
diff --git a/clang/test/SemaCXX/embed-init-list.cpp b/clang/test/SemaCXX/embed-init-list.cpp
index c511ca707a537d..d949afccb219f3 100644
--- a/clang/test/SemaCXX/embed-init-list.cpp
+++ b/clang/test/SemaCXX/embed-init-list.cpp
@@ -1,5 +1,4 @@
// RUN: %clang_cc1 -fsyntax-only -verify -Wno-c23-extensions %s
-// expected-no-diagnostics
namespace std {
typedef decltype(sizeof(int)) size_t;
@@ -13,6 +12,7 @@ template <class _E> class initializer_list {
public:
constexpr initializer_list() : __begin_(nullptr), __size_(0) {}
+ constexpr size_t size() const { return __size_; }
};
} // namespace std
@@ -69,3 +69,159 @@ void tests() {
#embed __FILE__
});
}
+
+namespace GH222796 {
+struct X {
+ template <typename... T>
+ constexpr X(T... t) : sum((0 + ... + int(t))), size(sizeof...(T)) {}
+ int sum;
+ int size;
+};
+
+constexpr X copy_list = {
+#embed __FILE__ limit(2)
+};
+static_assert(copy_list.size == 2, "");
+static_assert(copy_list.sum == '/' + '/', "");
+
+constexpr X direct_list{
+#embed __FILE__ limit(3)
+};
+static_assert(direct_list.size == 3, "");
+static_assert(direct_list.sum == '/' + '/' + ' ', "");
+
+constexpr X functional_cast() {
+ return X{
+#embed __FILE__ limit(2)
+ };
+}
+static_assert(functional_cast().size == 2, "");
+
+constexpr X returned() {
+ return {
+#embed __FILE__ limit(2)
+ };
+}
+static_assert(returned().size == 2, "");
+
+constexpr int arg(X x) { return x.size; }
+constexpr int arg(int) { return -1; }
+static_assert(arg({
+#embed __FILE__ limit(2)
+}) == 2, "");
+
+struct Member {
+ X m{
+#embed __FILE__ limit(2)
+ };
+};
+constexpr Member member{};
+static_assert(member.m.size == 2, "");
+
+constexpr X empty{
+#embed __FILE__ limit(0)
+};
+static_assert(empty.size == 0, "");
+
+struct Two {
+ Two(int, int);
+};
+void two() {
+ Two t = {
+#embed __FILE__ limit(2)
+ };
+ Two *p = new Two{
+#embed __FILE__ limit(2)
+ };
+ delete p;
+ (void)Two{
+#embed __FILE__ limit(2)
+ };
+}
+
+template <class... T> struct Deduced {
+ Deduced(T...);
+};
+Deduced deduced{
+#embed __FILE__ limit(2)
+};
+static_assert(__is_same(decltype(deduced), Deduced<int, int>), "");
+
+template <class T, unsigned N> constexpr unsigned bound(const T (&)[N]) {
+ return N;
+}
+static_assert(bound({
+#embed __FILE__ limit(3)
+}) == 3, "");
+
+template <class T> constexpr std::size_t il_size(std::initializer_list<T> l) {
+ return l.size();
+}
+static_assert(il_size({
+#embed __FILE__ limit(3)
+}) == 3, "");
+
+constexpr std::initializer_list<int> il = {
+#embed __FILE__ limit(3)
+};
+static_assert(il.size() == 3, "");
+
+constexpr const int &one_ref = {
+#embed __FILE__ limit(1)
+};
+static_assert(one_ref == '/', "");
+
+const int &ref = { // expected-error {{excess elements in scalar initializer}}
+#embed __FILE__ limit(2)
+};
+
+enum E : int { E1 };
+constexpr E one_enum{
+#embed __FILE__ limit(1)
+};
+static_assert(one_enum == '/', "");
+
+const E e{ // expected-error {{excess elements in scalar initializer}}
+#embed __FILE__ limit(2)
+};
+
+int scalar = { // expected-error {{excess elements in scalar initializer}}
+#embed __FILE__ limit(2)
+};
+
+constexpr _Complex double complex = {
+#embed __FILE__ limit(2)
+};
+static_assert(__real__ complex == '/' && __imag__ complex == '/', "");
+
+auto deduced_auto{ // expected-error {{excess elements in scalar initializer}}
+#embed __FILE__ limit(2)
+};
+
+void deduced_auto_expr() {
+ (void)new auto{ // expected-error {{excess elements in scalar initializer}}
+#embed __FILE__ limit(2)
+ };
+ (void)auto{ // expected-warning {{'auto' as a functional-style cast is a C++23 extension}} \
+ expected-error {{excess elements in scalar initializer}}
+#embed __FILE__ limit(2)
+ };
+}
+
+void scalar_arg(int); // expected-note {{candidate function not viable: cannot convert initializer list argument to 'int'}}
+void call_scalar_arg() {
+ scalar_arg({ // expected-error {{no matching function for call to 'scalar_arg'}}
+#embed __FILE__ limit(2)
+ });
+}
+
+_BitInt(7) fits[] = {
+#embed __FILE__ limit(3)
+};
+_BitInt(7) narrowed[] = { // expected-error at +1 {{constant expression evaluates to 82 which cannot be narrowed to type '_BitInt(7)'}}
+#embed __FILE__ limit(4)
+};
+std::initializer_list<_BitInt(7)> narrowed_il = { // expected-error at +1 {{constant expression evaluates to 82 which cannot be narrowed to type '_BitInt(7)'}}
+#embed __FILE__ limit(4)
+};
+}
>From 1e4abca67b2a57c8fd4be8870f0ec998bd34b38b Mon Sep 17 00:00:00 2001
From: Daniel Cederman <cederman at gaisler.com>
Date: Sat, 24 Oct 2026 06:20:50 +0200
Subject: [PATCH 04/37] [SPARC] Treat %g0 as a constant in the delay slot pass
(#234152)
This allows instructions such as clr, that reads %g0, to be moved past a
compare, that writes to %g0, and into a delay slot. %g0 is always 0, so
there is no dependency.
Assisted-by: Claude
---
llvm/lib/Target/Sparc/DelaySlotFiller.cpp | 2 +-
llvm/test/CodeGen/SPARC/2011-01-11-CC.ll | 6 +-
.../CodeGen/SPARC/2011-01-19-DelaySlot.ll | 62 +++++++++++++++++++
3 files changed, 65 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/Sparc/DelaySlotFiller.cpp b/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
index b5ae70ee36cac2..6edbc89f592e9a 100644
--- a/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
+++ b/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
@@ -334,7 +334,7 @@ void Filler::insertDefsUses(MachineBasicBlock::iterator MI,
continue;
Register Reg = MO.getReg();
- if (Reg == 0)
+ if (Reg == 0 || Subtarget->getRegisterInfo()->isConstantPhysReg(Reg))
continue;
if (MO.isDef())
RegDefs.insert(Reg);
diff --git a/llvm/test/CodeGen/SPARC/2011-01-11-CC.ll b/llvm/test/CodeGen/SPARC/2011-01-11-CC.ll
index e05c47bfee7665..9dffe0a3ce4561 100644
--- a/llvm/test/CodeGen/SPARC/2011-01-11-CC.ll
+++ b/llvm/test/CodeGen/SPARC/2011-01-11-CC.ll
@@ -25,15 +25,13 @@ define i32 @test_addx(i64 %a, i64 %b, i64 %c) nounwind {
; V8-NEXT: retl
; V8-NEXT: nop
; V8-NEXT: .LBB0_4: ! %entry
-; V8-NEXT: mov %g0, %o2
; V8-NEXT: cmp %o3, %o5
; V8-NEXT: bgu .LBB0_2
-; V8-NEXT: nop
+; V8-NEXT: mov %g0, %o2
; V8-NEXT: .LBB0_5: ! %entry
-; V8-NEXT: mov %g0, %o0
; V8-NEXT: cmp %o1, %o4
; V8-NEXT: be .LBB0_3
-; V8-NEXT: nop
+; V8-NEXT: mov %g0, %o0
; V8-NEXT: .LBB0_6: ! %entry
; V8-NEXT: retl
; V8-NEXT: mov %o2, %o0
diff --git a/llvm/test/CodeGen/SPARC/2011-01-19-DelaySlot.ll b/llvm/test/CodeGen/SPARC/2011-01-19-DelaySlot.ll
index 9863e9f743540f..5a96d87d8c6401 100644
--- a/llvm/test/CodeGen/SPARC/2011-01-19-DelaySlot.ll
+++ b/llvm/test/CodeGen/SPARC/2011-01-19-DelaySlot.ll
@@ -674,5 +674,67 @@ cont:
ret i32 %ret
}
+; cmp defines %g0, which must not keep the instructions before it that read %g0
+; out of the delay slot.
+define i32 @g0_use_before_cmp(i32 %a, i32 %b, i32 %c, i32 %d) #0 {
+; CHECK-LABEL: g0_use_before_cmp:
+; CHECK: ! %bb.0: ! %entry
+; CHECK-NEXT: cmp %o0, %o1
+; CHECK-NEXT: bgu .LBB17_2
+; CHECK-NEXT: nop
+; CHECK-NEXT: ! %bb.1: ! %entry
+; CHECK-NEXT: cmp %o2, %o3
+; CHECK-NEXT: bne .LBB17_3
+; CHECK-NEXT: mov %g0, %o0
+; CHECK-NEXT: ba .LBB17_4
+; CHECK-NEXT: nop
+; CHECK-NEXT: .LBB17_2:
+; CHECK-NEXT: cmp %o2, %o3
+; CHECK-NEXT: be .LBB17_4
+; CHECK-NEXT: mov 1, %o0
+; CHECK-NEXT: .LBB17_3: ! %entry
+; CHECK-NEXT: mov %g0, %o0
+; CHECK-NEXT: .LBB17_4: ! %entry
+; CHECK-NEXT: retl
+; CHECK-NEXT: nop
+;
+; UNOPT-LABEL: g0_use_before_cmp:
+; UNOPT: ! %bb.0: ! %entry
+; UNOPT-NEXT: add %sp, -112, %sp
+; UNOPT-NEXT: st %o3, [%sp+96] ! 4-byte Folded Spill
+; UNOPT-NEXT: st %o2, [%sp+100] ! 4-byte Folded Spill
+; UNOPT-NEXT: mov %o1, %o2
+; UNOPT-NEXT: mov %o0, %o1
+; UNOPT-NEXT: mov %g0, %o0
+; UNOPT-NEXT: st %o0, [%sp+104] ! 4-byte Folded Spill
+; UNOPT-NEXT: mov 1, %o0
+; UNOPT-NEXT: cmp %o1, %o2
+; UNOPT-NEXT: bgu .LBB17_2
+; UNOPT-NEXT: st %o0, [%sp+108]
+; UNOPT-NEXT: ! %bb.1: ! %entry
+; UNOPT-NEXT: ld [%sp+104], %o0 ! 4-byte Folded Reload
+; UNOPT-NEXT: st %o0, [%sp+108] ! 4-byte Folded Spill
+; UNOPT-NEXT: .LBB17_2: ! %entry
+; UNOPT-NEXT: ld [%sp+100], %o1 ! 4-byte Folded Reload
+; UNOPT-NEXT: ld [%sp+96], %o2 ! 4-byte Folded Reload
+; UNOPT-NEXT: ld [%sp+108], %o0 ! 4-byte Folded Reload
+; UNOPT-NEXT: cmp %o1, %o2
+; UNOPT-NEXT: be .LBB17_4
+; UNOPT-NEXT: st %o0, [%sp+92]
+; UNOPT-NEXT: ! %bb.3: ! %entry
+; UNOPT-NEXT: ld [%sp+104], %o0 ! 4-byte Folded Reload
+; UNOPT-NEXT: st %o0, [%sp+92] ! 4-byte Folded Spill
+; UNOPT-NEXT: .LBB17_4: ! %entry
+; UNOPT-NEXT: ld [%sp+92], %o0 ! 4-byte Folded Reload
+; UNOPT-NEXT: retl
+; UNOPT-NEXT: add %sp, 112, %sp
+entry:
+ %c1 = icmp ugt i32 %a, %b
+ %s1 = select i1 %c1, i32 1, i32 0
+ %c2 = icmp eq i32 %c, %d
+ %s2 = select i1 %c2, i32 %s1, i32 0
+ ret i32 %s2
+}
+
declare void @llvm.fake.use(...)
attributes #0 = { nounwind "disable-tail-calls"="true" }
>From 3f550f8b4a08ff3cfa94027f5abfa5fcd5d2a0d0 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder at redhat.com>
Date: Sat, 24 Oct 2026 06:27:54 +0200
Subject: [PATCH 05/37] [clang][bytecode][NFC] Use switch in
`Pointer::isWeak()` (#234914)
And save two lines.
---
clang/lib/AST/ByteCode/Pointer.h | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index 6c8ac0066f5977..0fc540cdd88586 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -919,23 +919,21 @@ class Pointer {
}
bool isWeak() const {
- if (isFunctionPointer()) {
+ switch (StorageKind) {
+ case Storage::Fn:
if (!Fn.Func || !Fn.Func->getDecl())
return false;
-
return Fn.Func->getDecl()->isWeak();
- }
-
- if (isOpaquePointer()) {
+ case Storage::Opaque:
if (const VarDecl *BaseDecl = Opaque.getBaseVarDecl())
return BaseDecl->isWeak();
return false;
- }
- if (!isBlockPointer())
+ case Storage::Block:
+ return BS.Pointee->isWeak();
+ default:
return false;
-
- assert(isBlockPointer());
- return BS.Pointee->isWeak();
+ }
+ llvm_unreachable("Unhandled storage kind");
}
/// Checks if the object is active.
bool isActive() const {
>From f8da8e372c0700ffca674c0c72404a27d0149cf8 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i at tianshilei.me>
Date: Fri, 23 Oct 2026 22:12:39 -0700
Subject: [PATCH 06/37] [AMDGPU] Remove XNACK from gfx12-5-generic (#233329)
This affects downstream targets.
---
clang/test/Driver/hip-sanitize-options.hip | 5 -
llvm/lib/Target/AMDGPU/GCNFeatures.td | 2 -
llvm/test/CodeGen/AMDGPU/sad.ll | 204 +++++++++------------
3 files changed, 82 insertions(+), 129 deletions(-)
diff --git a/clang/test/Driver/hip-sanitize-options.hip b/clang/test/Driver/hip-sanitize-options.hip
index 4c14df283f7753..95a0308677d33f 100644
--- a/clang/test/Driver/hip-sanitize-options.hip
+++ b/clang/test/Driver/hip-sanitize-options.hip
@@ -8,11 +8,6 @@
// RUN: -nogpuinc --rocm-path=%S/Inputs/rocm \
// RUN: %s 2>&1 | FileCheck -check-prefixes=NORDC %s
-// RUN: %clang -### --target=x86_64-unknown-linux-gnu --offload-arch=gfx12-5-generic:xnack+ \
-// RUN: -fsanitize=address \
-// RUN: -nogpuinc --rocm-path=%S/Inputs/rocm \
-// RUN: %s 2>&1 | FileCheck -check-prefixes=NORDC %s
-
// RUN: %clang -### --target=x86_64-unknown-linux-gnu --offload-arch=gfx900:xnack+ \
// RUN: -fsanitize=address -fno-gpu-sanitize \
// RUN: -nogpuinc --rocm-path=%S/Inputs/rocm \
diff --git a/llvm/lib/Target/AMDGPU/GCNFeatures.td b/llvm/lib/Target/AMDGPU/GCNFeatures.td
index a2760139eeebff..8a5d5d4c7c57d5 100644
--- a/llvm/lib/Target/AMDGPU/GCNFeatures.td
+++ b/llvm/lib/Target/AMDGPU/GCNFeatures.td
@@ -2705,8 +2705,6 @@ def FeatureISAVersion12_5_Generic: FeatureSet<
FeatureDataCacheLineSize128,
FeatureRequiresInitialUnclausedVmem,
FeatureNeedsTDMDrain,
- FeatureXNACKOnOffModes,
- FeatureSupportsXNACK,
])>;
def FeatureISAVersion13 : FeatureSet<
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index 3898a58d8cd271..5e31f9e463355f 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -35,13 +35,12 @@ define amdgpu_kernel void @v_sad_u32_pat1(ptr addrspace(1) %out, i32 %a, i32 %b,
;
; GFX12-5-SDAG-LABEL: v_sad_u32_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_min_u32 s3, s0, s1
; GFX12-5-SDAG-NEXT: s_max_u32 s0, s0, s1
@@ -50,18 +49,17 @@ define amdgpu_kernel void @v_sad_u32_pat1(ptr addrspace(1) %out, i32 %a, i32 %b,
; GFX12-5-SDAG-NEXT: s_add_co_i32 s0, s0, s2
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-5-SDAG-NEXT: global_store_b32 v0, v1, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_max_u32 s3, s0, s1
@@ -71,7 +69,7 @@ define amdgpu_kernel void @v_sad_u32_pat1(ptr addrspace(1) %out, i32 %a, i32 %b,
; GFX12-5-GISEL-NEXT: s_add_co_i32 s0, s0, s2
; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -116,7 +114,6 @@ define amdgpu_kernel void @v_sad_u32_constant_pat1(ptr addrspace(1) %out, i32 %a
;
; GFX12-5-SDAG-LABEL: v_sad_u32_constant_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -134,7 +131,6 @@ define amdgpu_kernel void @v_sad_u32_constant_pat1(ptr addrspace(1) %out, i32 %a
;
; GFX12-5-GISEL-LABEL: v_sad_u32_constant_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -196,13 +192,12 @@ define amdgpu_kernel void @v_sad_u32_pat2(ptr addrspace(1) %out, i32 %a, i32 %b,
;
; GFX12-5-SDAG-LABEL: v_sad_u32_pat2:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_min_u32 s3, s1, s0
; GFX12-5-SDAG-NEXT: s_max_u32 s0, s1, s0
@@ -211,29 +206,28 @@ define amdgpu_kernel void @v_sad_u32_pat2(ptr addrspace(1) %out, i32 %a, i32 %b,
; GFX12-5-SDAG-NEXT: s_add_co_i32 s0, s0, s2
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-5-SDAG-NEXT: global_store_b32 v0, v1, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_pat2:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_sub_co_u32 s3, s1, s0
-; GFX12-5-GISEL-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-5-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s0, s0, s1
-; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX12-5-GISEL-NEXT: s_cselect_b32 s0, s0, s3
; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: s_add_co_i32 s0, s0, s2
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%sub0 = sub i32 %a, %b
@@ -297,13 +291,12 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat1(ptr addrspace(1) %out, i
;
; GFX12-5-SDAG-LABEL: v_sad_u32_multi_use_sub_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_min_u32 s3, s0, s1
; GFX12-5-SDAG-NEXT: s_max_u32 s0, s0, s1
@@ -315,18 +308,17 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat1(ptr addrspace(1) %out, i
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v2, s1
; GFX12-5-SDAG-NEXT: scratch_store_b32 off, v1, s0 scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_multi_use_sub_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_max_u32 s3, s0, s1
@@ -338,7 +330,7 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat1(ptr addrspace(1) %out, i
; GFX12-5-GISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-5-GISEL-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -401,13 +393,12 @@ define amdgpu_kernel void @v_sad_u32_multi_use_add_pat1(ptr addrspace(1) %out, i
;
; GFX12-5-SDAG-LABEL: v_sad_u32_multi_use_add_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_min_u32 s3, s0, s1
@@ -419,18 +410,17 @@ define amdgpu_kernel void @v_sad_u32_multi_use_add_pat1(ptr addrspace(1) %out, i
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX12-5-SDAG-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX12-5-SDAG-NEXT: global_store_b32 v1, v0, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_multi_use_add_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_max_u32 s3, s0, s1
@@ -442,7 +432,7 @@ define amdgpu_kernel void @v_sad_u32_multi_use_add_pat1(ptr addrspace(1) %out, i
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX12-5-GISEL-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -507,13 +497,12 @@ define amdgpu_kernel void @v_sad_u32_multi_use_max_pat1(ptr addrspace(1) %out, i
;
; GFX12-5-SDAG-LABEL: v_sad_u32_multi_use_max_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_max_u32 s3, s0, s1
; GFX12-5-SDAG-NEXT: s_min_u32 s0, s0, s1
@@ -524,18 +513,17 @@ define amdgpu_kernel void @v_sad_u32_multi_use_max_pat1(ptr addrspace(1) %out, i
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v2, s0
; GFX12-5-SDAG-NEXT: scratch_store_b32 off, v1, s0 scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_multi_use_max_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_max_u32 s3, s0, s1
@@ -547,7 +535,7 @@ define amdgpu_kernel void @v_sad_u32_multi_use_max_pat1(ptr addrspace(1) %out, i
; GFX12-5-GISEL-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s0
; GFX12-5-GISEL-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -613,13 +601,12 @@ define amdgpu_kernel void @v_sad_u32_multi_use_min_pat1(ptr addrspace(1) %out, i
;
; GFX12-5-SDAG-LABEL: v_sad_u32_multi_use_min_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_min_u32 s3, s0, s1
; GFX12-5-SDAG-NEXT: s_max_u32 s0, s0, s1
@@ -630,18 +617,17 @@ define amdgpu_kernel void @v_sad_u32_multi_use_min_pat1(ptr addrspace(1) %out, i
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v2, s0
; GFX12-5-SDAG-NEXT: scratch_store_b32 off, v1, s0 scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_multi_use_min_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_max_u32 s3, s0, s1
@@ -653,7 +639,7 @@ define amdgpu_kernel void @v_sad_u32_multi_use_min_pat1(ptr addrspace(1) %out, i
; GFX12-5-GISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-5-GISEL-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -722,49 +708,47 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat2(ptr addrspace(1) %out, i
;
; GFX12-5-SDAG-LABEL: v_sad_u32_multi_use_sub_pat2:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_min_u32 s3, s1, s0
-; GFX12-5-SDAG-NEXT: s_max_u32 s4, s1, s0
+; GFX12-5-SDAG-NEXT: s_max_u32 s6, s1, s0
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s0, s0, s1
-; GFX12-5-SDAG-NEXT: s_sub_co_i32 s3, s4, s3
+; GFX12-5-SDAG-NEXT: s_sub_co_i32 s3, s6, s3
; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX12-5-SDAG-NEXT: s_add_co_i32 s1, s3, s2
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v2, s1
; GFX12-5-SDAG-NEXT: scratch_store_b32 off, v1, s0 scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_multi_use_sub_pat2:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_sub_co_u32 s3, s1, s0
-; GFX12-5-GISEL-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-5-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s0, s0, s1
-; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX12-5-GISEL-NEXT: s_cselect_b32 s1, s0, s3
; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: s_add_co_i32 s1, s1, s2
; GFX12-5-GISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-5-GISEL-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%sub0 = sub i32 %a, %b
@@ -831,13 +815,12 @@ define amdgpu_kernel void @v_sad_u32_multi_use_select_pat2(ptr addrspace(1) %out
;
; GFX12-5-SDAG-LABEL: v_sad_u32_multi_use_select_pat2:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_min_u32 s3, s1, s0
; GFX12-5-SDAG-NEXT: s_max_u32 s0, s1, s0
@@ -849,31 +832,30 @@ define amdgpu_kernel void @v_sad_u32_multi_use_select_pat2(ptr addrspace(1) %out
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v2, s1
; GFX12-5-SDAG-NEXT: scratch_store_b32 off, v1, s0 scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v0, v2, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_multi_use_select_pat2:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_sub_co_u32 s3, s1, s0
-; GFX12-5-GISEL-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-5-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s0, s0, s1
-; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX12-5-GISEL-NEXT: s_cselect_b32 s0, s0, s3
; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: s_add_co_i32 s1, s0, s2
; GFX12-5-GISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-5-GISEL-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v2, v1, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%sub0 = sub i32 %a, %b
@@ -944,68 +926,66 @@ define amdgpu_kernel void @v_sad_u32_vector_pat1(ptr addrspace(1) %out, <4 x i32
;
; GFX12-5-SDAG-LABEL: v_sad_u32_vector_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x2
; GFX12-5-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
; GFX12-5-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x54 nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: s_min_u32 s4, s8, s12
-; GFX12-5-SDAG-NEXT: s_max_u32 s5, s8, s12
+; GFX12-5-SDAG-NEXT: s_min_u32 s6, s8, s12
+; GFX12-5-SDAG-NEXT: s_max_u32 s7, s8, s12
; GFX12-5-SDAG-NEXT: s_min_u32 s8, s9, s13
; GFX12-5-SDAG-NEXT: s_max_u32 s9, s9, s13
; GFX12-5-SDAG-NEXT: s_min_u32 s12, s10, s14
; GFX12-5-SDAG-NEXT: s_max_u32 s10, s10, s14
; GFX12-5-SDAG-NEXT: s_min_u32 s13, s11, s15
; GFX12-5-SDAG-NEXT: s_max_u32 s11, s11, s15
-; GFX12-5-SDAG-NEXT: s_sub_co_i32 s4, s5, s4
-; GFX12-5-SDAG-NEXT: s_sub_co_i32 s5, s9, s8
+; GFX12-5-SDAG-NEXT: s_sub_co_i32 s6, s7, s6
+; GFX12-5-SDAG-NEXT: s_sub_co_i32 s7, s9, s8
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s8, s11, s13
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s9, s10, s12
; GFX12-5-SDAG-NEXT: s_add_co_i32 s8, s8, s3
; GFX12-5-SDAG-NEXT: s_add_co_i32 s9, s9, s2
-; GFX12-5-SDAG-NEXT: s_add_co_i32 s4, s4, s0
-; GFX12-5-SDAG-NEXT: s_add_co_i32 s5, s5, s1
-; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s4
-; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s9
+; GFX12-5-SDAG-NEXT: s_add_co_i32 s6, s6, s0
+; GFX12-5-SDAG-NEXT: s_add_co_i32 s7, s7, s1
+; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s6
+; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s9
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v3, s8
-; GFX12-5-SDAG-NEXT: global_store_b128 v4, v[0:3], s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_vector_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x2
; GFX12-5-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
; GFX12-5-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x54 nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-GISEL-NEXT: s_max_u32 s4, s8, s12
-; GFX12-5-GISEL-NEXT: s_max_u32 s5, s9, s13
+; GFX12-5-GISEL-NEXT: s_max_u32 s6, s8, s12
+; GFX12-5-GISEL-NEXT: s_max_u32 s7, s9, s13
; GFX12-5-GISEL-NEXT: s_max_u32 s16, s10, s14
; GFX12-5-GISEL-NEXT: s_max_u32 s17, s11, s15
; GFX12-5-GISEL-NEXT: s_min_u32 s8, s8, s12
; GFX12-5-GISEL-NEXT: s_min_u32 s9, s9, s13
; GFX12-5-GISEL-NEXT: s_min_u32 s10, s10, s14
; GFX12-5-GISEL-NEXT: s_min_u32 s11, s11, s15
-; GFX12-5-GISEL-NEXT: s_sub_co_i32 s4, s4, s8
-; GFX12-5-GISEL-NEXT: s_sub_co_i32 s5, s5, s9
+; GFX12-5-GISEL-NEXT: s_sub_co_i32 s6, s6, s8
+; GFX12-5-GISEL-NEXT: s_sub_co_i32 s7, s7, s9
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s8, s16, s10
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s9, s17, s11
-; GFX12-5-GISEL-NEXT: s_add_co_i32 s0, s4, s0
-; GFX12-5-GISEL-NEXT: s_add_co_i32 s1, s5, s1
+; GFX12-5-GISEL-NEXT: s_add_co_i32 s0, s6, s0
+; GFX12-5-GISEL-NEXT: s_add_co_i32 s1, s7, s1
; GFX12-5-GISEL-NEXT: s_add_co_i32 s2, s8, s2
; GFX12-5-GISEL-NEXT: s_add_co_i32 s3, s9, s3
; GFX12-5-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX12-5-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX12-5-GISEL-NEXT: global_store_b128 v4, v[0:3], s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt <4 x i32> %a, %b
%t0 = select <4 x i1> %icmp0, <4 x i32> %a, <4 x i32> %b
@@ -1089,40 +1069,38 @@ define amdgpu_kernel void @v_sad_u32_vector_pat2(ptr addrspace(1) %out, <4 x i32
;
; GFX12-5-SDAG-LABEL: v_sad_u32_vector_pat2:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x2
; GFX12-5-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
; GFX12-5-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x54 nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: s_min_u32 s4, s8, s12
-; GFX12-5-SDAG-NEXT: s_max_u32 s5, s8, s12
+; GFX12-5-SDAG-NEXT: s_min_u32 s6, s8, s12
+; GFX12-5-SDAG-NEXT: s_max_u32 s7, s8, s12
; GFX12-5-SDAG-NEXT: s_min_u32 s8, s9, s13
; GFX12-5-SDAG-NEXT: s_max_u32 s9, s9, s13
; GFX12-5-SDAG-NEXT: s_min_u32 s12, s10, s14
; GFX12-5-SDAG-NEXT: s_max_u32 s10, s10, s14
; GFX12-5-SDAG-NEXT: s_min_u32 s13, s11, s15
; GFX12-5-SDAG-NEXT: s_max_u32 s11, s11, s15
-; GFX12-5-SDAG-NEXT: s_sub_co_i32 s4, s5, s4
-; GFX12-5-SDAG-NEXT: s_sub_co_i32 s5, s9, s8
+; GFX12-5-SDAG-NEXT: s_sub_co_i32 s6, s7, s6
+; GFX12-5-SDAG-NEXT: s_sub_co_i32 s7, s9, s8
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s8, s11, s13
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s9, s10, s12
; GFX12-5-SDAG-NEXT: s_add_co_i32 s8, s8, s3
; GFX12-5-SDAG-NEXT: s_add_co_i32 s9, s9, s2
-; GFX12-5-SDAG-NEXT: s_add_co_i32 s4, s4, s0
-; GFX12-5-SDAG-NEXT: s_add_co_i32 s5, s5, s1
-; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s4
-; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s9
+; GFX12-5-SDAG-NEXT: s_add_co_i32 s6, s6, s0
+; GFX12-5-SDAG-NEXT: s_add_co_i32 s7, s7, s1
+; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s6
+; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s9
; GFX12-5-SDAG-NEXT: v_mov_b32_e32 v3, s8
-; GFX12-5-SDAG-NEXT: global_store_b128 v4, v[0:3], s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_vector_pat2:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -1210,7 +1188,6 @@ define amdgpu_kernel void @v_sad_u32_i16_pat1(ptr addrspace(1) %out, i16 %a, i16
;
; GFX12-5-SDAG-LABEL: v_sad_u32_i16_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -1231,7 +1208,6 @@ define amdgpu_kernel void @v_sad_u32_i16_pat1(ptr addrspace(1) %out, i16 %a, i16
;
; GFX12-5-GISEL-LABEL: v_sad_u32_i16_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -1307,14 +1283,12 @@ define amdgpu_kernel void @v_sad_u32_i16_pat2(ptr addrspace(1) %out) {
;
; GFX12-5-SDAG-LABEL: v_sad_u32_i16_pat2:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-5-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX12-5-SDAG-NEXT: s_wait_xcnt 0x0
; GFX12-5-SDAG-NEXT: global_load_u16 v2, v[0:1], off scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-5-SDAG-NEXT: global_load_u16 v3, v[0:1], off scope:SCOPE_SYS
@@ -1333,14 +1307,12 @@ define amdgpu_kernel void @v_sad_u32_i16_pat2(ptr addrspace(1) %out) {
;
; GFX12-5-GISEL-LABEL: v_sad_u32_i16_pat2:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-5-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX12-5-GISEL-NEXT: s_wait_xcnt 0x0
; GFX12-5-GISEL-NEXT: global_load_u16 v2, v[0:1], off scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-5-GISEL-NEXT: global_load_u16 v3, v[0:1], off scope:SCOPE_SYS
@@ -1415,7 +1387,6 @@ define amdgpu_kernel void @v_sad_u32_i8_pat1(ptr addrspace(1) %out, i8 %a, i8 %b
;
; GFX12-5-SDAG-LABEL: v_sad_u32_i8_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -1436,7 +1407,6 @@ define amdgpu_kernel void @v_sad_u32_i8_pat1(ptr addrspace(1) %out, i8 %a, i8 %b
;
; GFX12-5-GISEL-LABEL: v_sad_u32_i8_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -1514,14 +1484,12 @@ define amdgpu_kernel void @v_sad_u32_i8_pat2(ptr addrspace(1) %out) {
;
; GFX12-5-SDAG-LABEL: v_sad_u32_i8_pat2:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-5-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX12-5-SDAG-NEXT: s_wait_xcnt 0x0
; GFX12-5-SDAG-NEXT: global_load_u8 v2, v[0:1], off scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-5-SDAG-NEXT: global_load_u8 v3, v[0:1], off scope:SCOPE_SYS
@@ -1537,14 +1505,12 @@ define amdgpu_kernel void @v_sad_u32_i8_pat2(ptr addrspace(1) %out) {
;
; GFX12-5-GISEL-LABEL: v_sad_u32_i8_pat2:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-5-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX12-5-GISEL-NEXT: s_wait_xcnt 0x0
; GFX12-5-GISEL-NEXT: global_load_u8 v2, v[0:1], off scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-5-GISEL-NEXT: global_load_u8 v3, v[0:1], off scope:SCOPE_SYS
@@ -1620,7 +1586,6 @@ define amdgpu_kernel void @s_sad_u32_i8_pat2(ptr addrspace(1) %out, i8 zeroext %
;
; GFX12-5-SDAG-LABEL: s_sad_u32_i8_pat2:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -1641,7 +1606,6 @@ define amdgpu_kernel void @s_sad_u32_i8_pat2(ptr addrspace(1) %out, i8 zeroext %
;
; GFX12-5-GISEL-LABEL: s_sad_u32_i8_pat2:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
@@ -1710,45 +1674,43 @@ define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat1(ptr addrspace(1) %
;
; GFX12-5-SDAG-LABEL: v_sad_u32_mismatched_operands_pat1:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: s_max_u32 s4, s0, s1
+; GFX12-5-SDAG-NEXT: s_max_u32 s6, s0, s1
; GFX12-5-SDAG-NEXT: s_cmp_le_u32 s0, s1
; GFX12-5-SDAG-NEXT: s_cselect_b32 s0, s0, s3
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-5-SDAG-NEXT: s_sub_co_i32 s0, s4, s0
+; GFX12-5-SDAG-NEXT: s_sub_co_i32 s0, s6, s0
; GFX12-5-SDAG-NEXT: s_add_co_i32 s0, s0, s2
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-5-SDAG-NEXT: global_store_b32 v0, v1, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_mismatched_operands_pat1:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-GISEL-NEXT: s_max_u32 s4, s0, s1
+; GFX12-5-GISEL-NEXT: s_max_u32 s6, s0, s1
; GFX12-5-GISEL-NEXT: s_cmp_le_u32 s0, s1
; GFX12-5-GISEL-NEXT: s_cselect_b32 s0, s0, s3
; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-5-GISEL-NEXT: s_sub_co_i32 s0, s4, s0
+; GFX12-5-GISEL-NEXT: s_sub_co_i32 s0, s6, s0
; GFX12-5-GISEL-NEXT: s_add_co_i32 s0, s0, s2
; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -1799,42 +1761,40 @@ define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat2(ptr addrspace(1) %
;
; GFX12-5-SDAG-LABEL: v_sad_u32_mismatched_operands_pat2:
; GFX12-5-SDAG: ; %bb.0:
-; GFX12-5-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-SDAG-NEXT: v_nop
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_clause 0x1
; GFX12-5-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
-; GFX12-5-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s3, s0, s3
; GFX12-5-SDAG-NEXT: v_sub_co_u32 v0, s0, s1, s0
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-5-SDAG-NEXT: v_cndmask_b32_e64 v0, v0, s3, s0
; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
-; GFX12-5-SDAG-NEXT: global_store_b32 v1, v0, s[6:7]
+; GFX12-5-SDAG-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_mismatched_operands_pat2:
; GFX12-5-GISEL: ; %bb.0:
-; GFX12-5-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-5-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX12-5-GISEL-NEXT: v_nop
; GFX12-5-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-GISEL-NEXT: s_clause 0x1
; GFX12-5-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
-; GFX12-5-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX12-5-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-5-GISEL-NEXT: s_sub_co_u32 s1, s1, s0
-; GFX12-5-GISEL-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-5-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s0, s0, s3
-; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX12-5-GISEL-NEXT: s_cselect_b32 s0, s0, s1
; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: s_add_co_i32 s0, s0, s2
; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[6:7]
+; GFX12-5-GISEL-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-5-GISEL-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%sub0 = sub i32 %a, %d
>From b0e824572198bd0474480998708f25ebcdebcc73 Mon Sep 17 00:00:00 2001
From: Hongyu Chen <xxs_chy at outlook.com>
Date: Sat, 24 Oct 2026 13:17:59 +0800
Subject: [PATCH 07/37] [RISCV][P-ext] Support Packed Slide (#234629)
This patch supports packed slide intrinsics using funnel shifts.
On RV32, the 64-bit funnel shift expands into select instructions, so it
involves branches.
---
clang/lib/Headers/riscv_packed_simd.h | 44 ++
clang/test/CodeGen/RISCV/rvp-intrinsics.c | 391 ++++++++++++++++++
.../riscv_packed_simd.c | 124 ++++++
llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 90 ++++
llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 204 +++++++++
5 files changed, 853 insertions(+)
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index 88c330cf7761d7..89f678a63e94e6 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -179,6 +179,24 @@ typedef uint32_t uint32x2_t __attribute__((__vector_size__(8)));
return __builtin_elementwise_fshr(__rs1, __rd, (ty)__shamt); \
}
+#define __packed_slideupx(name, ty, bits_ty, log2_elt_bits) \
+ static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rd, ty __rs1, \
+ unsigned __rs2) { \
+ bits_ty __bits = __builtin_elementwise_fshl( \
+ __builtin_bit_cast(bits_ty, __rd), __builtin_bit_cast(bits_ty, __rs1), \
+ (bits_ty)__rs2 << (log2_elt_bits)); \
+ return __builtin_bit_cast(ty, __bits); \
+ }
+
+#define __packed_slidedownx(name, ty, bits_ty, log2_elt_bits) \
+ static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rd, ty __rs1, \
+ unsigned __rs2) { \
+ bits_ty __bits = __builtin_elementwise_fshr( \
+ __builtin_bit_cast(bits_ty, __rs1), __builtin_bit_cast(bits_ty, __rd), \
+ (bits_ty)__rs2 << (log2_elt_bits)); \
+ return __builtin_bit_cast(ty, __bits); \
+ }
+
#define __packed_widen_convert(name, rty, ty) \
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
return __builtin_convertvector(__rs1, rty); \
@@ -1473,6 +1491,30 @@ __packed_slide1(pslide1down_u16x4, uint16x4_t, uint16_t, 1, 2, 3, 4)
__packed_slide1(pslide1down_i32x2, int32x2_t, int32_t, 1, 2)
__packed_slide1(pslide1down_u32x2, uint32x2_t, uint32_t, 1, 2)
+/* Packed Slide (32-bit) */
+__packed_slideupx(pslideupx_i8x4, int8x4_t, uint32_t, 3)
+__packed_slideupx(pslideupx_u8x4, uint8x4_t, uint32_t, 3)
+__packed_slideupx(pslideupx_i16x2, int16x2_t, uint32_t, 4)
+__packed_slideupx(pslideupx_u16x2, uint16x2_t, uint32_t, 4)
+__packed_slidedownx(pslidedownx_i8x4, int8x4_t, uint32_t, 3)
+__packed_slidedownx(pslidedownx_u8x4, uint8x4_t, uint32_t, 3)
+__packed_slidedownx(pslidedownx_i16x2, int16x2_t, uint32_t, 4)
+__packed_slidedownx(pslidedownx_u16x2, uint16x2_t, uint32_t, 4)
+
+/* Packed Slide (64-bit) */
+__packed_slideupx(pslideupx_i8x8, int8x8_t, uint64_t, 3)
+__packed_slideupx(pslideupx_u8x8, uint8x8_t, uint64_t, 3)
+__packed_slideupx(pslideupx_i16x4, int16x4_t, uint64_t, 4)
+__packed_slideupx(pslideupx_u16x4, uint16x4_t, uint64_t, 4)
+__packed_slideupx(pslideupx_i32x2, int32x2_t, uint64_t, 5)
+__packed_slideupx(pslideupx_u32x2, uint32x2_t, uint64_t, 5)
+__packed_slidedownx(pslidedownx_i8x8, int8x8_t, uint64_t, 3)
+__packed_slidedownx(pslidedownx_u8x8, uint8x8_t, uint64_t, 3)
+__packed_slidedownx(pslidedownx_i16x4, int16x4_t, uint64_t, 4)
+__packed_slidedownx(pslidedownx_u16x4, uint16x4_t, uint64_t, 4)
+__packed_slidedownx(pslidedownx_i32x2, int32x2_t, uint64_t, 5)
+__packed_slidedownx(pslidedownx_u32x2, uint32x2_t, uint64_t, 5)
+
/* Packed Subvector Extract */
__packed_subvector_extract8(pget_i8x8_i8x4, int8x4_t, int8x8_t)
__packed_subvector_extract8(pget_u8x8_u8x4, uint8x4_t, uint8x8_t)
@@ -1674,6 +1716,8 @@ __packed_reinterpret(u32x2_i32x2, int32x2_t, uint32x2_t)
#undef __packed_cls
#undef __packed_slx
#undef __packed_srx
+#undef __packed_slideupx
+#undef __packed_slidedownx
#undef __packed_widen_convert
#undef __packed_widen_binary_op
#undef __packed_widen_binary_acc_op
diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
index 928520c11c0a4e..ea42d98eec04c1 100644
--- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c
+++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
@@ -10302,6 +10302,397 @@ int64_t test_pm2waddasu_i64(int64_t rd, int16x2_t rs1, uint16x2_t rs2) {
return __riscv_pm2waddasu_i64(rd, rs1, rs2);
}
+/* Packed Slide (32-bit) */
+
+// RV32-LABEL: define dso_local i32 @test_pslideupx_i8x4(
+// RV32-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 3
+// RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[RD_COERCE]], i32 [[RS1_COERCE]], i32 [[SHL_I]])
+// RV32-NEXT: ret i32 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i32 @test_pslideupx_i8x4(
+// RV64-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 3
+// RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[RD_COERCE]], i32 [[RS1_COERCE]], i32 [[SHL_I]])
+// RV64-NEXT: ret i32 [[TMP0]]
+//
+int8x4_t test_pslideupx_i8x4(int8x4_t rd, int8x4_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i8x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pslideupx_u8x4(
+// RV32-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 3
+// RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[RD_COERCE]], i32 [[RS1_COERCE]], i32 [[SHL_I]])
+// RV32-NEXT: ret i32 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i32 @test_pslideupx_u8x4(
+// RV64-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 3
+// RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[RD_COERCE]], i32 [[RS1_COERCE]], i32 [[SHL_I]])
+// RV64-NEXT: ret i32 [[TMP0]]
+//
+uint8x4_t test_pslideupx_u8x4(uint8x4_t rd, uint8x4_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u8x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pslideupx_i16x2(
+// RV32-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 4
+// RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[RD_COERCE]], i32 [[RS1_COERCE]], i32 [[SHL_I]])
+// RV32-NEXT: ret i32 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i32 @test_pslideupx_i16x2(
+// RV64-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 4
+// RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[RD_COERCE]], i32 [[RS1_COERCE]], i32 [[SHL_I]])
+// RV64-NEXT: ret i32 [[TMP0]]
+//
+int16x2_t test_pslideupx_i16x2(int16x2_t rd, int16x2_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pslideupx_u16x2(
+// RV32-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 4
+// RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[RD_COERCE]], i32 [[RS1_COERCE]], i32 [[SHL_I]])
+// RV32-NEXT: ret i32 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i32 @test_pslideupx_u16x2(
+// RV64-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 4
+// RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[RD_COERCE]], i32 [[RS1_COERCE]], i32 [[SHL_I]])
+// RV64-NEXT: ret i32 [[TMP0]]
+//
+uint16x2_t test_pslideupx_u16x2(uint16x2_t rd, uint16x2_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pslidedownx_i8x4(
+// RV32-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 3
+// RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[RS1_COERCE]], i32 [[RD_COERCE]], i32 [[SHL_I]])
+// RV32-NEXT: ret i32 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i32 @test_pslidedownx_i8x4(
+// RV64-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 3
+// RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[RS1_COERCE]], i32 [[RD_COERCE]], i32 [[SHL_I]])
+// RV64-NEXT: ret i32 [[TMP0]]
+//
+int8x4_t test_pslidedownx_i8x4(int8x4_t rd, int8x4_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i8x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pslidedownx_u8x4(
+// RV32-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 3
+// RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[RS1_COERCE]], i32 [[RD_COERCE]], i32 [[SHL_I]])
+// RV32-NEXT: ret i32 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i32 @test_pslidedownx_u8x4(
+// RV64-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 3
+// RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[RS1_COERCE]], i32 [[RD_COERCE]], i32 [[SHL_I]])
+// RV64-NEXT: ret i32 [[TMP0]]
+//
+uint8x4_t test_pslidedownx_u8x4(uint8x4_t rd, uint8x4_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_u8x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pslidedownx_i16x2(
+// RV32-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 4
+// RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[RS1_COERCE]], i32 [[RD_COERCE]], i32 [[SHL_I]])
+// RV32-NEXT: ret i32 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i32 @test_pslidedownx_i16x2(
+// RV64-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 4
+// RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[RS1_COERCE]], i32 [[RD_COERCE]], i32 [[SHL_I]])
+// RV64-NEXT: ret i32 [[TMP0]]
+//
+int16x2_t test_pslidedownx_i16x2(int16x2_t rd, int16x2_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pslidedownx_u16x2(
+// RV32-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 4
+// RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[RS1_COERCE]], i32 [[RD_COERCE]], i32 [[SHL_I]])
+// RV32-NEXT: ret i32 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i32 @test_pslidedownx_u16x2(
+// RV64-SAME: i32 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[SHL_I:%.*]] = shl i32 [[RS2]], 4
+// RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[RS1_COERCE]], i32 [[RD_COERCE]], i32 [[SHL_I]])
+// RV64-NEXT: ret i32 [[TMP0]]
+//
+uint16x2_t test_pslidedownx_u16x2(uint16x2_t rd, uint16x2_t rs1,
+ unsigned rs2) {
+ return __riscv_pslidedownx_u16x2(rd, rs1, rs2);
+}
+
+/* Packed Slide (64-bit) */
+
+// RV32-LABEL: define dso_local i64 @test_pslideupx_i8x8(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 3
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslideupx_i8x8(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 3
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+int8x8_t test_pslideupx_i8x8(int8x8_t rd, int8x8_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i8x8(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslideupx_u8x8(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 3
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslideupx_u8x8(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 3
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+uint8x8_t test_pslideupx_u8x8(uint8x8_t rd, uint8x8_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u8x8(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslideupx_i16x4(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 4
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslideupx_i16x4(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 4
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+int16x4_t test_pslideupx_i16x4(int16x4_t rd, int16x4_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i16x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslideupx_u16x4(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 4
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslideupx_u16x4(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 4
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+uint16x4_t test_pslideupx_u16x4(uint16x4_t rd, uint16x4_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u16x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslideupx_i32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 5
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslideupx_i32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 5
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+int32x2_t test_pslideupx_i32x2(int32x2_t rd, int32x2_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslideupx_u32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 5
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslideupx_u32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 5
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshl.i64(i64 [[RD_COERCE]], i64 [[RS1_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+uint32x2_t test_pslideupx_u32x2(uint32x2_t rd, uint32x2_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslidedownx_i8x8(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 3
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslidedownx_i8x8(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 3
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+int8x8_t test_pslidedownx_i8x8(int8x8_t rd, int8x8_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i8x8(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslidedownx_u8x8(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 3
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslidedownx_u8x8(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 3
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+uint8x8_t test_pslidedownx_u8x8(uint8x8_t rd, uint8x8_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_u8x8(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslidedownx_i16x4(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 4
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslidedownx_i16x4(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 4
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+int16x4_t test_pslidedownx_i16x4(int16x4_t rd, int16x4_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i16x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslidedownx_u16x4(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 4
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslidedownx_u16x4(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 4
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+uint16x4_t test_pslidedownx_u16x4(uint16x4_t rd, uint16x4_t rs1,
+ unsigned rs2) {
+ return __riscv_pslidedownx_u16x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslidedownx_i32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 5
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslidedownx_i32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 5
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+int32x2_t test_pslidedownx_i32x2(int32x2_t rd, int32x2_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pslidedownx_u32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV32-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 5
+// RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV32-NEXT: ret i64 [[TMP0]]
+//
+// RV64-LABEL: define dso_local i64 @test_pslidedownx_u32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i32 noundef signext [[RS2:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[CONV_I:%.*]] = zext i32 [[RS2]] to i64
+// RV64-NEXT: [[SHL_I:%.*]] = shl nuw nsw i64 [[CONV_I]], 5
+// RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.fshr.i64(i64 [[RS1_COERCE]], i64 [[RD_COERCE]], i64 [[SHL_I]])
+// RV64-NEXT: ret i64 [[TMP0]]
+//
+uint32x2_t test_pslidedownx_u32x2(uint32x2_t rd, uint32x2_t rs1,
+ unsigned rs2) {
+ return __riscv_pslidedownx_u32x2(rd, rs1, rs2);
+}
+
/* Packed Byte Multiplication with Horizontal Addition and Accumulate */
// CHECK-LABEL: define dso_local i64 @test_pm4adda_i8x8(
diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
index 134f2f51598075..365af7c993a5f6 100644
--- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
+++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
@@ -6128,3 +6128,127 @@ int32x2_t test_pslide1down_i32x2(int32x2_t rd, int32_t rs1) {
uint32x2_t test_pslide1down_u32x2(uint32x2_t rd, uint32_t rs1) {
return __riscv_pslide1down_u32x2(rd, rs1);
}
+
+/* Packed Slide (32-bit) */
+
+// CHECK-LABEL: test_pslideupx_i8x4:
+// CHECK: slx
+int8x4_t test_pslideupx_i8x4(int8x4_t rd, int8x4_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i8x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslideupx_u8x4:
+// CHECK: slx
+uint8x4_t test_pslideupx_u8x4(uint8x4_t rd, uint8x4_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u8x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslideupx_i16x2:
+// CHECK: slx
+int16x2_t test_pslideupx_i16x2(int16x2_t rd, int16x2_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslideupx_u16x2:
+// CHECK: slx
+uint16x2_t test_pslideupx_u16x2(uint16x2_t rd, uint16x2_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_i8x4:
+// CHECK: srx
+int8x4_t test_pslidedownx_i8x4(int8x4_t rd, int8x4_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i8x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_u8x4:
+// CHECK: srx
+uint8x4_t test_pslidedownx_u8x4(uint8x4_t rd, uint8x4_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_u8x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_i16x2:
+// CHECK: srx
+int16x2_t test_pslidedownx_i16x2(int16x2_t rd, int16x2_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_u16x2:
+// CHECK: srx
+uint16x2_t test_pslidedownx_u16x2(uint16x2_t rd, uint16x2_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_u16x2(rd, rs1, rs2);
+}
+
+/* Packed Slide (64-bit) */
+
+// CHECK-LABEL: test_pslideupx_i8x8:
+// CHECK: slx
+int8x8_t test_pslideupx_i8x8(int8x8_t rd, int8x8_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i8x8(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslideupx_u8x8:
+// CHECK: slx
+uint8x8_t test_pslideupx_u8x8(uint8x8_t rd, uint8x8_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u8x8(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslideupx_i16x4:
+// CHECK: slx
+int16x4_t test_pslideupx_i16x4(int16x4_t rd, int16x4_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslideupx_u16x4:
+// CHECK: slx
+uint16x4_t test_pslideupx_u16x4(uint16x4_t rd, uint16x4_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslideupx_i32x2:
+// RV64: slx
+int32x2_t test_pslideupx_i32x2(int32x2_t rd, int32x2_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslideupx_u32x2:
+// RV64: slx
+uint32x2_t test_pslideupx_u32x2(uint32x2_t rd, uint32x2_t rs1, unsigned rs2) {
+ return __riscv_pslideupx_u32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_i8x8:
+// CHECK: srx
+int8x8_t test_pslidedownx_i8x8(int8x8_t rd, int8x8_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i8x8(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_u8x8:
+// CHECK: srx
+uint8x8_t test_pslidedownx_u8x8(uint8x8_t rd, uint8x8_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_u8x8(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_i16x4:
+// CHECK: srx
+int16x4_t test_pslidedownx_i16x4(int16x4_t rd, int16x4_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_u16x4:
+// CHECK: srx
+uint16x4_t test_pslidedownx_u16x4(uint16x4_t rd, uint16x4_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_u16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_i32x2:
+// RV64: srx
+int32x2_t test_pslidedownx_i32x2(int32x2_t rd, int32x2_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pslidedownx_u32x2:
+// RV64: srx
+uint32x2_t test_pslidedownx_u32x2(uint32x2_t rd, uint32x2_t rs1, unsigned rs2) {
+ return __riscv_pslidedownx_u32x2(rd, rs1, rs2);
+}
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
index ac29235202b411..fc49df7ea62795 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
@@ -4150,3 +4150,93 @@ define <2 x i16> @test_pslide1down_v2i16(<2 x i16> %rd, i16 %rs1) {
%res = shufflevector <2 x i16> %rd, <2 x i16> %scalar, <2 x i32> <i32 1, i32 2>
ret <2 x i16> %res
}
+
+define <4 x i8> @test_pslideupx_v4i8(<4 x i8> %rd, <4 x i8> %rs1, i32 %rs2) {
+; RV32-LABEL: test_pslideupx_v4i8:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a2, a2, 3
+; RV32-NEXT: slx a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslideupx_v4i8:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 3
+; RV64-NEXT: slli a1, a1, 32
+; RV64-NEXT: andi a2, a2, 24
+; RV64-NEXT: slx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <4 x i8> %rd to i32
+ %rs1.bits = bitcast <4 x i8> %rs1 to i32
+ %shamt = shl i32 %rs2, 3
+ %res.bits = call i32 @llvm.fshl.i32(i32 %rd.bits, i32 %rs1.bits, i32 %shamt)
+ %res = bitcast i32 %res.bits to <4 x i8>
+ ret <4 x i8> %res
+}
+
+define <4 x i8> @test_pslidedownx_v4i8(<4 x i8> %rd, <4 x i8> %rs1, i32 %rs2) {
+; RV32-LABEL: test_pslidedownx_v4i8:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a2, a2, 3
+; RV32-NEXT: srx a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslidedownx_v4i8:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 3
+; RV64-NEXT: slli a0, a0, 32
+; RV64-NEXT: ori a2, a2, 32
+; RV64-NEXT: srx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <4 x i8> %rd to i32
+ %rs1.bits = bitcast <4 x i8> %rs1 to i32
+ %shamt = shl i32 %rs2, 3
+ %res.bits = call i32 @llvm.fshr.i32(i32 %rs1.bits, i32 %rd.bits, i32 %shamt)
+ %res = bitcast i32 %res.bits to <4 x i8>
+ ret <4 x i8> %res
+}
+
+define <2 x i16> @test_pslideupx_v2i16(<2 x i16> %rd, <2 x i16> %rs1,
+ i32 %rs2) {
+; RV32-LABEL: test_pslideupx_v2i16:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a2, a2, 4
+; RV32-NEXT: slx a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslideupx_v2i16:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 4
+; RV64-NEXT: slli a1, a1, 32
+; RV64-NEXT: andi a2, a2, 16
+; RV64-NEXT: slx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <2 x i16> %rd to i32
+ %rs1.bits = bitcast <2 x i16> %rs1 to i32
+ %shamt = shl i32 %rs2, 4
+ %res.bits = call i32 @llvm.fshl.i32(i32 %rd.bits, i32 %rs1.bits, i32 %shamt)
+ %res = bitcast i32 %res.bits to <2 x i16>
+ ret <2 x i16> %res
+}
+
+define <2 x i16> @test_pslidedownx_v2i16(<2 x i16> %rd, <2 x i16> %rs1,
+ i32 %rs2) {
+; RV32-LABEL: test_pslidedownx_v2i16:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a2, a2, 4
+; RV32-NEXT: srx a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslidedownx_v2i16:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 4
+; RV64-NEXT: slli a0, a0, 32
+; RV64-NEXT: ori a2, a2, 32
+; RV64-NEXT: srx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <2 x i16> %rd to i32
+ %rs1.bits = bitcast <2 x i16> %rs1 to i32
+ %shamt = shl i32 %rs2, 4
+ %res.bits = call i32 @llvm.fshr.i32(i32 %rs1.bits, i32 %rd.bits, i32 %shamt)
+ %res = bitcast i32 %res.bits to <2 x i16>
+ ret <2 x i16> %res
+}
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index 8e190a181400fd..45d3e90405f4c2 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -9084,3 +9084,207 @@ define <2 x i32> @test_pslide1down_v2i32(<2 x i32> %rd, i32 %rs1) {
%res = shufflevector <2 x i32> %rd, <2 x i32> %scalar, <2 x i32> <i32 1, i32 2>
ret <2 x i32> %res
}
+
+define <8 x i8> @test_pslideupx_v8i8(<8 x i8> %rd, <8 x i8> %rs1, i32 %rs2) {
+; RV32-LABEL: test_pslideupx_v8i8:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a5, a0
+; RV32-NEXT: slli a0, a4, 29
+; RV32-NEXT: srli a6, a0, 31
+; RV32-NEXT: slli a4, a4, 3
+; RV32-NEXT: bnez a6, .LBB536_2
+; RV32-NEXT: # %bb.1:
+; RV32-NEXT: mv a0, a5
+; RV32-NEXT: mv a5, a1
+; RV32-NEXT: slx a5, a0, a4
+; RV32-NEXT: beqz a6, .LBB536_3
+; RV32-NEXT: j .LBB536_4
+; RV32-NEXT: .LBB536_2:
+; RV32-NEXT: mv a0, a3
+; RV32-NEXT: slx a5, a3, a4
+; RV32-NEXT: bnez a6, .LBB536_4
+; RV32-NEXT: .LBB536_3:
+; RV32-NEXT: mv a2, a3
+; RV32-NEXT: .LBB536_4:
+; RV32-NEXT: slx a0, a2, a4
+; RV32-NEXT: mv a1, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslideupx_v8i8:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 3
+; RV64-NEXT: slx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <8 x i8> %rd to i64
+ %rs1.bits = bitcast <8 x i8> %rs1 to i64
+ %rs2.ext = zext i32 %rs2 to i64
+ %shamt = shl i64 %rs2.ext, 3
+ %res.bits = call i64 @llvm.fshl.i64(i64 %rd.bits, i64 %rs1.bits, i64 %shamt)
+ %res = bitcast i64 %res.bits to <8 x i8>
+ ret <8 x i8> %res
+}
+
+define <8 x i8> @test_pslidedownx_v8i8(<8 x i8> %rd, <8 x i8> %rs1,
+ i32 %rs2) {
+; RV32-LABEL: test_pslidedownx_v8i8:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a5, a4, 3
+; RV32-NEXT: andi a6, a5, 32
+; RV32-NEXT: bnez a6, .LBB537_2
+; RV32-NEXT: # %bb.1:
+; RV32-NEXT: mv a3, a2
+; RV32-NEXT: mv a2, a1
+; RV32-NEXT: .LBB537_2:
+; RV32-NEXT: mv a4, a2
+; RV32-NEXT: srx a4, a3, a5
+; RV32-NEXT: beqz a6, .LBB537_4
+; RV32-NEXT: # %bb.3:
+; RV32-NEXT: mv a0, a1
+; RV32-NEXT: .LBB537_4:
+; RV32-NEXT: srx a0, a2, a5
+; RV32-NEXT: mv a1, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslidedownx_v8i8:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 3
+; RV64-NEXT: srx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <8 x i8> %rd to i64
+ %rs1.bits = bitcast <8 x i8> %rs1 to i64
+ %rs2.ext = zext i32 %rs2 to i64
+ %shamt = shl i64 %rs2.ext, 3
+ %res.bits = call i64 @llvm.fshr.i64(i64 %rs1.bits, i64 %rd.bits, i64 %shamt)
+ %res = bitcast i64 %res.bits to <8 x i8>
+ ret <8 x i8> %res
+}
+
+define <4 x i16> @test_pslideupx_v4i16(<4 x i16> %rd, <4 x i16> %rs1,
+ i32 %rs2) {
+; RV32-LABEL: test_pslideupx_v4i16:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a5, a0
+; RV32-NEXT: slli a0, a4, 30
+; RV32-NEXT: srli a6, a0, 31
+; RV32-NEXT: slli a4, a4, 4
+; RV32-NEXT: bnez a6, .LBB538_2
+; RV32-NEXT: # %bb.1:
+; RV32-NEXT: mv a0, a5
+; RV32-NEXT: mv a5, a1
+; RV32-NEXT: slx a5, a0, a4
+; RV32-NEXT: beqz a6, .LBB538_3
+; RV32-NEXT: j .LBB538_4
+; RV32-NEXT: .LBB538_2:
+; RV32-NEXT: mv a0, a3
+; RV32-NEXT: slx a5, a3, a4
+; RV32-NEXT: bnez a6, .LBB538_4
+; RV32-NEXT: .LBB538_3:
+; RV32-NEXT: mv a2, a3
+; RV32-NEXT: .LBB538_4:
+; RV32-NEXT: slx a0, a2, a4
+; RV32-NEXT: mv a1, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslideupx_v4i16:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 4
+; RV64-NEXT: slx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <4 x i16> %rd to i64
+ %rs1.bits = bitcast <4 x i16> %rs1 to i64
+ %rs2.ext = zext i32 %rs2 to i64
+ %shamt = shl i64 %rs2.ext, 4
+ %res.bits = call i64 @llvm.fshl.i64(i64 %rd.bits, i64 %rs1.bits, i64 %shamt)
+ %res = bitcast i64 %res.bits to <4 x i16>
+ ret <4 x i16> %res
+}
+
+define <4 x i16> @test_pslidedownx_v4i16(<4 x i16> %rd, <4 x i16> %rs1,
+ i32 %rs2) {
+; RV32-LABEL: test_pslidedownx_v4i16:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a5, a4, 4
+; RV32-NEXT: andi a6, a5, 32
+; RV32-NEXT: bnez a6, .LBB539_2
+; RV32-NEXT: # %bb.1:
+; RV32-NEXT: mv a3, a2
+; RV32-NEXT: mv a2, a1
+; RV32-NEXT: .LBB539_2:
+; RV32-NEXT: mv a4, a2
+; RV32-NEXT: srx a4, a3, a5
+; RV32-NEXT: beqz a6, .LBB539_4
+; RV32-NEXT: # %bb.3:
+; RV32-NEXT: mv a0, a1
+; RV32-NEXT: .LBB539_4:
+; RV32-NEXT: srx a0, a2, a5
+; RV32-NEXT: mv a1, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslidedownx_v4i16:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 4
+; RV64-NEXT: srx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <4 x i16> %rd to i64
+ %rs1.bits = bitcast <4 x i16> %rs1 to i64
+ %rs2.ext = zext i32 %rs2 to i64
+ %shamt = shl i64 %rs2.ext, 4
+ %res.bits = call i64 @llvm.fshr.i64(i64 %rs1.bits, i64 %rd.bits, i64 %shamt)
+ %res = bitcast i64 %res.bits to <4 x i16>
+ ret <4 x i16> %res
+}
+
+define <2 x i32> @test_pslideupx_v2i32(<2 x i32> %rd, <2 x i32> %rs1,
+ i32 %rs2) {
+; RV32-LABEL: test_pslideupx_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: andi a4, a4, 1
+; RV32-NEXT: beqz a4, .LBB540_2
+; RV32-NEXT: # %bb.1:
+; RV32-NEXT: mv a1, a0
+; RV32-NEXT: mv a0, a3
+; RV32-NEXT: .LBB540_2:
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslideupx_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 5
+; RV64-NEXT: slx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <2 x i32> %rd to i64
+ %rs1.bits = bitcast <2 x i32> %rs1 to i64
+ %rs2.ext = zext i32 %rs2 to i64
+ %shamt = shl i64 %rs2.ext, 5
+ %res.bits = call i64 @llvm.fshl.i64(i64 %rd.bits, i64 %rs1.bits, i64 %shamt)
+ %res = bitcast i64 %res.bits to <2 x i32>
+ ret <2 x i32> %res
+}
+
+define <2 x i32> @test_pslidedownx_v2i32(<2 x i32> %rd, <2 x i32> %rs1,
+ i32 %rs2) {
+; RV32-LABEL: test_pslidedownx_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a4, a4, 31
+; RV32-NEXT: srli a4, a4, 26
+; RV32-NEXT: bnez a4, .LBB541_2
+; RV32-NEXT: # %bb.1:
+; RV32-NEXT: mv a2, a1
+; RV32-NEXT: mv a1, a0
+; RV32-NEXT: .LBB541_2:
+; RV32-NEXT: mv a0, a1
+; RV32-NEXT: mv a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pslidedownx_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a2, 5
+; RV64-NEXT: srx a0, a1, a2
+; RV64-NEXT: ret
+ %rd.bits = bitcast <2 x i32> %rd to i64
+ %rs1.bits = bitcast <2 x i32> %rs1 to i64
+ %rs2.ext = zext i32 %rs2 to i64
+ %shamt = shl i64 %rs2.ext, 5
+ %res.bits = call i64 @llvm.fshr.i64(i64 %rs1.bits, i64 %rd.bits, i64 %shamt)
+ %res = bitcast i64 %res.bits to <2 x i32>
+ ret <2 x i32> %res
+}
>From dc3e2624849a453d7961384d9dd2cb07a38e29dd Mon Sep 17 00:00:00 2001
From: Reid Kleckner <rkleckner at nvidia.com>
Date: Fri, 23 Oct 2026 22:28:47 -0700
Subject: [PATCH 08/37] [TailCallElim] Place byval copy-back after trailing
instructions (#234699)
Fix the insertion point of the byval copy operation we insert when
updating our incoming byval argument when transforming tail calls into
loops. We should insert the argument update memcpy at the point of the
**return**, not the call. There may be side-effect free instructions
between the call and the return, and they may read the incoming
argument, and using the incorrect insertion point results in a
miscompile.
Fixes #234334
LLM-aided
---
.../Scalar/TailRecursionElimination.cpp | 13 +--
.../TailCallElim/tre-byval-parameter-2.ll | 4 +-
.../TailCallElim/tre-byval-parameter.ll | 2 +-
.../TailCallElim/tre-byval-trailing-load.ll | 87 +++++++++++++++++++
4 files changed, 98 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Transforms/TailCallElim/tre-byval-trailing-load.ll
diff --git a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
index 9bbe253a923f35..30097e86b06c69 100644
--- a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
@@ -555,7 +555,8 @@ class TailRecursionEliminator {
void copyByValueOperandIntoLocalTemp(CallInst *CI, int OpndIdx);
- void copyLocalTempOfByValueOperandIntoArguments(CallInst *CI, int OpndIdx);
+ void copyLocalTempOfByValueOperandIntoArguments(CallInst *CI, int OpndIdx,
+ Instruction *InsertPt);
public:
static bool eliminate(Function &F, const TargetTransformInfo *TTI,
@@ -829,9 +830,11 @@ void TailRecursionEliminator::copyByValueOperandIntoLocalTemp(CallInst *CI,
}
// Creates a copy from temporarily variable(keeping value of ByVal argument)
-// into the corresponding function argument location.
+// into the corresponding function argument location. The copy is inserted
+// before InsertPt, which must come after any instructions that followed the
+// call, since those may still read the current value of the argument.
void TailRecursionEliminator::copyLocalTempOfByValueOperandIntoArguments(
- CallInst *CI, int OpndIdx) {
+ CallInst *CI, int OpndIdx, Instruction *InsertPt) {
Type *AggTy = CI->getParamByValType(OpndIdx);
assert(AggTy);
const DataLayout &DL = F.getDataLayout();
@@ -839,7 +842,7 @@ void TailRecursionEliminator::copyLocalTempOfByValueOperandIntoArguments(
// Get alignment of byVal operand.
Align Alignment(CI->getParamAlign(OpndIdx).valueOrOne());
- IRBuilder<> Builder(CI);
+ IRBuilder<> Builder(InsertPt);
Value *Size = Builder.getInt64(DL.getTypeAllocSize(AggTy));
// Copy data from the temporarily variable into corresponding
@@ -908,7 +911,7 @@ bool TailRecursionEliminator::eliminateCall(CallInst *CI) {
// parameters passed into the tail-recursive call.
for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
if (CI->isByValArgument(I)) {
- copyLocalTempOfByValueOperandIntoArguments(CI, I);
+ copyLocalTempOfByValueOperandIntoArguments(CI, I, Ret);
// When eliminating a tail call, we modify the values of the arguments.
// Therefore, if the byval parameter has a readonly attribute, we have to
// remove it. It is safe because, from the perspective of a caller, the
diff --git a/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll b/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
index fa771ad01f3c38..d5b46b622b9d1c 100644
--- a/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
+++ b/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
@@ -51,10 +51,10 @@ define dso_local void @_Z7dostuff1AS_i(ptr nocapture byval(%struct.A) align 8 %a
; CHECK-NEXT: [[ADD]] = add nsw i32 [[I_TR]], 1
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP1]], ptr align 8 [[AGG_TMP]], i64 80, i1 false)
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP52]], ptr align 8 [[AGG_TMP5]], i64 80, i1 false)
-; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[A]], ptr align 8 [[AGG_TMP1]], i64 80, i1 false)
-; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[B]], ptr align 8 [[AGG_TMP52]], i64 80, i1 false)
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[AGG_TMP]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[AGG_TMP5]])
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[A]], ptr align 8 [[AGG_TMP1]], i64 80, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[B]], ptr align 8 [[AGG_TMP52]], i64 80, i1 false)
; CHECK-NEXT: br label [[TAILRECURSE]]
; CHECK: return:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll b/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
index dedd0811bea514..f0e581e968a035 100644
--- a/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
+++ b/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
@@ -49,9 +49,9 @@ define dso_local i32 @_Z3fooi1S(i32 %count, ptr nocapture readonly byval(%struct
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(20) [[AGG_TMP14]], ptr nonnull align 8 dereferenceable(20) [[AGG_TMP1]], i64 20, i1 false)
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(20) [[AGG_TMP_I]], ptr nonnull align 8 dereferenceable(20) [[AGG_TMP14]], i64 20, i1 false)
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP_I1]], ptr align 8 [[AGG_TMP_I]], i64 20, i1 false)
-; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[P1]], ptr align 8 [[AGG_TMP_I1]], i64 20, i1 false)
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[AGG_TMP14]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[AGG_TMP_I]])
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[P1]], ptr align 8 [[AGG_TMP_I1]], i64 20, i1 false)
; CHECK-NEXT: br label [[TAILRECURSE]]
; CHECK: return:
; CHECK-NEXT: ret i32 [[CALL]]
diff --git a/llvm/test/Transforms/TailCallElim/tre-byval-trailing-load.ll b/llvm/test/Transforms/TailCallElim/tre-byval-trailing-load.ll
new file mode 100644
index 00000000000000..795591deae358b
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/tre-byval-trailing-load.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=tailcallelim -verify-dom-info -S | FileCheck %s
+
+; A load of the byval argument after the self-call must see the caller's value,
+; so the copy of the next iteration's value into the argument has to come after
+; it.
+
+%S = type { i32, i32 }
+
+ at g = global %S { i32 7, i32 0 }
+
+define i32 @f(ptr byval(%S) align 4 %p, i32 %n) {
+; CHECK-LABEL: define i32 @f(
+; CHECK-SAME: ptr byval([[S:%.*]]) align 4 [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[G:%.*]] = alloca [[S]], align 1
+; CHECK-NEXT: br label %[[TAILRECURSE:.*]]
+; CHECK: [[TAILRECURSE]]:
+; CHECK-NEXT: [[ACCUMULATOR_TR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[S:%.*]], %[[REC:.*]] ]
+; CHECK-NEXT: [[N_TR:%.*]] = phi i32 [ [[N]], %[[ENTRY]] ], [ 0, %[[REC]] ]
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[N_TR]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[BASE:.*]], label %[[REC]]
+; CHECK: [[BASE]]:
+; CHECK-NEXT: [[BV:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[ACCUMULATOR_RET_TR:%.*]] = add i32 [[ACCUMULATOR_TR]], [[BV]]
+; CHECK-NEXT: ret i32 [[ACCUMULATOR_RET_TR]]
+; CHECK: [[REC]]:
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[G]], ptr align 1 @g, i64 8, i1 false)
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[S]] = add i32 [[ACCUMULATOR_TR]], [[V]]
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[P]], ptr align 1 [[G]], i64 8, i1 false)
+; CHECK-NEXT: br label %[[TAILRECURSE]]
+;
+entry:
+ %c = icmp eq i32 %n, 0
+ br i1 %c, label %base, label %rec
+
+base:
+ %bv = load i32, ptr %p, align 4
+ ret i32 %bv
+
+rec:
+ %r = call i32 @f(ptr byval(%S) @g, i32 0)
+ %v = load i32, ptr %p, align 4
+ %s = add i32 %r, %v
+ ret i32 %s
+}
+
+; The trailing load goes through a GEP of the byval argument.
+define i32 @f_gep(ptr byval(%S) align 4 %p, i32 %n) {
+; CHECK-LABEL: define i32 @f_gep(
+; CHECK-SAME: ptr byval([[S:%.*]]) align 4 [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[G:%.*]] = alloca [[S]], align 1
+; CHECK-NEXT: br label %[[TAILRECURSE:.*]]
+; CHECK: [[TAILRECURSE]]:
+; CHECK-NEXT: [[ACCUMULATOR_TR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[S:%.*]], %[[REC:.*]] ]
+; CHECK-NEXT: [[N_TR:%.*]] = phi i32 [ [[N]], %[[ENTRY]] ], [ 0, %[[REC]] ]
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[N_TR]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[BASE:.*]], label %[[REC]]
+; CHECK: [[BASE]]:
+; CHECK-NEXT: [[BV:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[ACCUMULATOR_RET_TR:%.*]] = add i32 [[ACCUMULATOR_TR]], [[BV]]
+; CHECK-NEXT: ret i32 [[ACCUMULATOR_RET_TR]]
+; CHECK: [[REC]]:
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[G]], ptr align 1 @g, i64 8, i1 false)
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[S]] = add i32 [[ACCUMULATOR_TR]], [[V]]
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[P]], ptr align 1 [[G]], i64 8, i1 false)
+; CHECK-NEXT: br label %[[TAILRECURSE]]
+;
+entry:
+ %c = icmp eq i32 %n, 0
+ br i1 %c, label %base, label %rec
+
+base:
+ %bv = load i32, ptr %p, align 4
+ ret i32 %bv
+
+rec:
+ %r = call i32 @f_gep(ptr byval(%S) @g, i32 0)
+ %gep = getelementptr inbounds i8, ptr %p, i64 4
+ %v = load i32, ptr %gep, align 4
+ %s = add i32 %r, %v
+ ret i32 %s
+}
>From fc78f4fa1965b2f54cad895475941a912bf42177 Mon Sep 17 00:00:00 2001
From: Jez Ng <me at jezng.com>
Date: Fri, 23 Oct 2026 23:03:04 -0700
Subject: [PATCH 09/37] [lld-macho] Fix export trie for names containing 0xff
bytes (#233399)
On signed-char hosts, charAt() promotes a 0xff byte to -1, aliasing the
end-of-string sentinel used by the three-way radix sort. As a result, an
export could be silently dropped or truncated. Fix is to convert the
byte to unsigned char before promoting it to int. I also verified that
ld-prime handles 0xff perfectly fine.
Assisted by Opus 5.5
---------
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
lld/MachO/ExportTrie.cpp | 4 ++-
lld/test/MachO/export-trie-high-bytes.s | 33 +++++++++++++++++++++++++
2 files changed, 36 insertions(+), 1 deletion(-)
create mode 100644 lld/test/MachO/export-trie-high-bytes.s
diff --git a/lld/MachO/ExportTrie.cpp b/lld/MachO/ExportTrie.cpp
index a3780d6fdc2fd7..4a9e14c3d920d8 100644
--- a/lld/MachO/ExportTrie.cpp
+++ b/lld/MachO/ExportTrie.cpp
@@ -200,11 +200,13 @@ TrieNode *TrieBuilder::makeNode() {
return node;
}
+// Like operator[], but returns -1 when pos is past the end of the name. -1
+// sorts below every byte, so "_foo" sorts before "_foobar".
static int charAt(const Symbol *sym, size_t pos) {
StringRef str = sym->getName();
if (pos >= str.size())
return -1;
- return str[pos];
+ return static_cast<unsigned char>(str[pos]);
}
// Build the trie by performing a three-way radix quicksort: We start by sorting
diff --git a/lld/test/MachO/export-trie-high-bytes.s b/lld/test/MachO/export-trie-high-bytes.s
new file mode 100644
index 00000000000000..cf8a96df8271a1
--- /dev/null
+++ b/lld/test/MachO/export-trie-high-bytes.s
@@ -0,0 +1,33 @@
+# REQUIRES: x86
+
+## Regression test for export names containing a 0xff byte, which as a signed
+## char equals -1, the value the trie builder uses for end-of-string. _a sits
+## right before the 0xff in _a\xffb, and _c\xff ends with it. 0x80 is the most
+## negative signed char and shares its _d prefix with an ASCII name.
+
+## The assembly below writes the bytes as <FF> and <80>, and sed swaps in the
+## raw bytes. printf generates the sed script because BSD sed has no \xff.
+# RUN: printf 's/<FF>/\377/g\ns/<80>/\200/g\n' > %t.sed
+# RUN: sed -f %t.sed %s | llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t.o
+# RUN: %lld -dylib %t.o -o %t.dylib
+# RUN: llvm-objdump --macho --exports-trie %t.dylib > %t.txt
+
+## cat -v prints 0xff as M-^? and 0x80 as M-^@, so the checks below can name
+## the bytes in plain ASCII. (lit's builtin cat only does this for files, not
+## stdin, hence %t.txt.)
+# RUN: cat -v %t.txt | FileCheck %s
+
+# CHECK-LABEL: Exports trie:
+# CHECK-DAG: _a{{$}}
+# CHECK-DAG: _aM-^?b{{$}}
+# CHECK-DAG: _cM-^?{{$}}
+# CHECK-DAG: _dM-^@x{{$}}
+# CHECK-DAG: _dy{{$}}
+
+.globl _a, "_a<FF>b", "_c<FF>", "_d<80>x", _dy
+_a:
+"_a<FF>b":
+"_c<FF>":
+"_d<80>x":
+_dy:
+ ret
>From a4bd6bd2e68184b38f828ca8ec51e5cd40b4f64c Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Fri, 23 Oct 2026 23:12:36 -0700
Subject: [PATCH 10/37] AArch64: Fix missing ... function separators in mir
test (#235119)
update_mir_test_checks misbehaves when the end of function marker
is missing.
---
.../AArch64/aarch64-combine-gather-lanes.mir | 252 ++++++++++--------
1 file changed, 138 insertions(+), 114 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir
index e6d07de52ac7e5..93989ae9440138 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir
+++ b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir
@@ -8,19 +8,21 @@ body: |
liveins: $x0, $x1, $x2, $x3, $x4
; CHECK-LABEL: name: split_loads_to_fpr128
- ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK: liveins: $x0, $x1, $x2, $x3, $x4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
- ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], [[COPY1]], 0, 1
- ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LD_i32]], %subreg.ssub
- ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, [[COPY2]]
- ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY3]], 0
- ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LD1_0]], %subreg.ssub
- ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, [[COPY4]]
- ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]]
- ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+ ; CHECK-NEXT: [[LDRSroX:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], [[COPY1]], 0, 1
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LDRSroX]], %subreg.ssub
+ ; CHECK-NEXT: [[LD1i32_:%[0-9]+]]:fpr128 = LD1i32 [[SUBREG_TO_REG]], 1, [[COPY2]]
+ ; CHECK-NEXT: [[LDRSui:%[0-9]+]]:fpr32 = LDRSui [[COPY3]], 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LDRSui]], %subreg.ssub
+ ; CHECK-NEXT: [[LD1i32_1:%[0-9]+]]:fpr128 = LD1i32 [[SUBREG_TO_REG1]], 1, [[COPY4]]
+ ; CHECK-NEXT: [[ZIP1v2i64_:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD1i32_]], [[LD1i32_1]]
+ ; CHECK-NEXT: $q0 = COPY [[ZIP1v2i64_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
@@ -34,7 +36,7 @@ body: |
%9:fpr128 = LD1i32 %8, 3, %4
$q0 = COPY %9
RET_ReallyLR implicit $q0
-
+...
---
name: split_loads_to_fpr128_ui
body: |
@@ -42,19 +44,21 @@ body: |
liveins: $x0, $x1, $x2, $x3, $x4
; CHECK-LABEL: name: split_loads_to_fpr128_ui
- ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK: liveins: $x0, $x1, $x2, $x3, $x4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
- ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSui [[COPY]], 0
- ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LD_i32]], %subreg.ssub
- ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, [[COPY1]]
- ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY2]], 0
- ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LD1_0]], %subreg.ssub
- ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, [[COPY3]]
- ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]]
- ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+ ; CHECK-NEXT: [[LDRSui:%[0-9]+]]:fpr32 = LDRSui [[COPY]], 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LDRSui]], %subreg.ssub
+ ; CHECK-NEXT: [[LD1i32_:%[0-9]+]]:fpr128 = LD1i32 [[SUBREG_TO_REG]], 1, [[COPY1]]
+ ; CHECK-NEXT: [[LDRSui1:%[0-9]+]]:fpr32 = LDRSui [[COPY2]], 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LDRSui1]], %subreg.ssub
+ ; CHECK-NEXT: [[LD1i32_1:%[0-9]+]]:fpr128 = LD1i32 [[SUBREG_TO_REG1]], 1, [[COPY3]]
+ ; CHECK-NEXT: [[ZIP1v2i64_:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD1i32_]], [[LD1i32_1]]
+ ; CHECK-NEXT: $q0 = COPY [[ZIP1v2i64_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
@@ -68,7 +72,7 @@ body: |
%9:fpr128 = LD1i32 %8, 3, killed %3
$q0 = COPY %9
RET_ReallyLR implicit $q0
-
+...
---
name: split_loads_to_fpr128_i16
body: |
@@ -76,7 +80,9 @@ body: |
liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8
; CHECK-LABEL: name: split_loads_to_fpr128_i16
- ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
@@ -85,18 +91,18 @@ body: |
; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64common = COPY $x6
; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr64common = COPY $x7
; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr64common = COPY $x8
- ; CHECK-NEXT: [[LD_i16:%[0-9]+]]:fpr16 = LDRHroX [[COPY]], killed [[COPY1]], 0, 1
- ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LD_i16]], %subreg.hsub
- ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i16 [[FIRST_REG]], 1, [[COPY2]]
- ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i16 [[LD0_1]], 2, [[COPY3]]
- ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i16 [[LD0_2]], 3, [[COPY4]]
- ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr16 = LDRHui [[COPY5]], 0
- ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LD1_0]], %subreg.hsub
- ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i16 [[SECOND_REG]], 1, [[COPY6]]
- ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i16 [[LD1_1]], 2, [[COPY7]]
- ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i16 [[LD1_2]], 3, [[COPY8]]
- ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_3]], [[LD1_3]]
- ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+ ; CHECK-NEXT: [[LDRHroX:%[0-9]+]]:fpr16 = LDRHroX [[COPY]], killed [[COPY1]], 0, 1
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LDRHroX]], %subreg.hsub
+ ; CHECK-NEXT: [[LD1i16_:%[0-9]+]]:fpr128 = LD1i16 [[SUBREG_TO_REG]], 1, [[COPY2]]
+ ; CHECK-NEXT: [[LD1i16_1:%[0-9]+]]:fpr128 = LD1i16 [[LD1i16_]], 2, [[COPY3]]
+ ; CHECK-NEXT: [[LD1i16_2:%[0-9]+]]:fpr128 = LD1i16 [[LD1i16_1]], 3, [[COPY4]]
+ ; CHECK-NEXT: [[LDRHui:%[0-9]+]]:fpr16 = LDRHui [[COPY5]], 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LDRHui]], %subreg.hsub
+ ; CHECK-NEXT: [[LD1i16_3:%[0-9]+]]:fpr128 = LD1i16 [[SUBREG_TO_REG1]], 1, [[COPY6]]
+ ; CHECK-NEXT: [[LD1i16_4:%[0-9]+]]:fpr128 = LD1i16 [[LD1i16_3]], 2, [[COPY7]]
+ ; CHECK-NEXT: [[LD1i16_5:%[0-9]+]]:fpr128 = LD1i16 [[LD1i16_4]], 3, [[COPY8]]
+ ; CHECK-NEXT: [[ZIP1v2i64_:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD1i16_2]], [[LD1i16_5]]
+ ; CHECK-NEXT: $q0 = COPY [[ZIP1v2i64_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
@@ -118,7 +124,7 @@ body: |
%17:fpr128 = LD1i16 %16, 7, killed %8
$q0 = COPY %17
RET_ReallyLR implicit $q0
-
+...
---
name: split_loads_to_fpr128_i16_ui
body: |
@@ -126,7 +132,9 @@ body: |
liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8
; CHECK-LABEL: name: split_loads_to_fpr128_i16_ui
- ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
@@ -135,18 +143,18 @@ body: |
; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64common = COPY $x6
; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr64common = COPY $x7
; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr64common = COPY $x8
- ; CHECK-NEXT: [[LD_i16:%[0-9]+]]:fpr16 = LDRHui [[COPY]], 0
- ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LD_i16]], %subreg.hsub
- ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i16 [[FIRST_REG]], 1, [[COPY1]]
- ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i16 [[LD0_1]], 2, [[COPY2]]
- ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i16 [[LD0_2]], 3, [[COPY3]]
- ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr16 = LDRHui [[COPY4]], 0
- ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LD1_0]], %subreg.hsub
- ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i16 [[SECOND_REG]], 1, [[COPY5]]
- ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i16 [[LD1_1]], 2, [[COPY6]]
- ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i16 [[LD1_2]], 3, [[COPY7]]
- ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_3]], [[LD1_3]]
- ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+ ; CHECK-NEXT: [[LDRHui:%[0-9]+]]:fpr16 = LDRHui [[COPY]], 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LDRHui]], %subreg.hsub
+ ; CHECK-NEXT: [[LD1i16_:%[0-9]+]]:fpr128 = LD1i16 [[SUBREG_TO_REG]], 1, [[COPY1]]
+ ; CHECK-NEXT: [[LD1i16_1:%[0-9]+]]:fpr128 = LD1i16 [[LD1i16_]], 2, [[COPY2]]
+ ; CHECK-NEXT: [[LD1i16_2:%[0-9]+]]:fpr128 = LD1i16 [[LD1i16_1]], 3, [[COPY3]]
+ ; CHECK-NEXT: [[LDRHui1:%[0-9]+]]:fpr16 = LDRHui [[COPY4]], 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LDRHui1]], %subreg.hsub
+ ; CHECK-NEXT: [[LD1i16_3:%[0-9]+]]:fpr128 = LD1i16 [[SUBREG_TO_REG1]], 1, [[COPY5]]
+ ; CHECK-NEXT: [[LD1i16_4:%[0-9]+]]:fpr128 = LD1i16 [[LD1i16_3]], 2, [[COPY6]]
+ ; CHECK-NEXT: [[LD1i16_5:%[0-9]+]]:fpr128 = LD1i16 [[LD1i16_4]], 3, [[COPY7]]
+ ; CHECK-NEXT: [[ZIP1v2i64_:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD1i16_2]], [[LD1i16_5]]
+ ; CHECK-NEXT: $q0 = COPY [[ZIP1v2i64_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
@@ -168,7 +176,7 @@ body: |
%17:fpr128 = LD1i16 %16, 7, killed %7
$q0 = COPY %17
RET_ReallyLR implicit $q0
-
+...
---
name: split_loads_to_fpr128_i8
body: |
@@ -176,7 +184,9 @@ body: |
liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16
; CHECK-LABEL: name: split_loads_to_fpr128_i8
- ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
@@ -193,26 +203,26 @@ body: |
; CHECK-NEXT: [[COPY14:%[0-9]+]]:gpr64common = COPY $x14
; CHECK-NEXT: [[COPY15:%[0-9]+]]:gpr64common = COPY $x15
; CHECK-NEXT: [[COPY16:%[0-9]+]]:gpr64common = COPY $x16
- ; CHECK-NEXT: [[LD_i8:%[0-9]+]]:fpr8 = LDRBroX [[COPY]], killed [[COPY1]], 0, 0
- ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LD_i8]], %subreg.bsub
- ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i8 [[FIRST_REG]], 1, [[COPY2]]
- ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i8 [[LD0_1]], 2, [[COPY3]]
- ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i8 [[LD0_2]], 3, [[COPY4]]
- ; CHECK-NEXT: [[LD0_4:%[0-9]+]]:fpr128 = LD1i8 [[LD0_3]], 4, [[COPY5]]
- ; CHECK-NEXT: [[LD0_5:%[0-9]+]]:fpr128 = LD1i8 [[LD0_4]], 5, [[COPY6]]
- ; CHECK-NEXT: [[LD0_6:%[0-9]+]]:fpr128 = LD1i8 [[LD0_5]], 6, [[COPY7]]
- ; CHECK-NEXT: [[LD0_7:%[0-9]+]]:fpr128 = LD1i8 [[LD0_6]], 7, [[COPY8]]
- ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr8 = LDRBui [[COPY9]], 0
- ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LD1_0]], %subreg.bsub
- ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i8 [[SECOND_REG]], 1, [[COPY10]]
- ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i8 [[LD1_1]], 2, [[COPY11]]
- ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i8 [[LD1_2]], 3, [[COPY12]]
- ; CHECK-NEXT: [[LD1_4:%[0-9]+]]:fpr128 = LD1i8 [[LD1_3]], 4, [[COPY13]]
- ; CHECK-NEXT: [[LD1_5:%[0-9]+]]:fpr128 = LD1i8 [[LD1_4]], 5, [[COPY14]]
- ; CHECK-NEXT: [[LD1_6:%[0-9]+]]:fpr128 = LD1i8 [[LD1_5]], 6, [[COPY15]]
- ; CHECK-NEXT: [[LD1_7:%[0-9]+]]:fpr128 = LD1i8 [[LD1_6]], 7, [[COPY16]]
- ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_7]], [[LD1_7]]
- ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+ ; CHECK-NEXT: [[LDRBroX:%[0-9]+]]:fpr8 = LDRBroX [[COPY]], killed [[COPY1]], 0, 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LDRBroX]], %subreg.bsub
+ ; CHECK-NEXT: [[LD1i8_:%[0-9]+]]:fpr128 = LD1i8 [[SUBREG_TO_REG]], 1, [[COPY2]]
+ ; CHECK-NEXT: [[LD1i8_1:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_]], 2, [[COPY3]]
+ ; CHECK-NEXT: [[LD1i8_2:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_1]], 3, [[COPY4]]
+ ; CHECK-NEXT: [[LD1i8_3:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_2]], 4, [[COPY5]]
+ ; CHECK-NEXT: [[LD1i8_4:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_3]], 5, [[COPY6]]
+ ; CHECK-NEXT: [[LD1i8_5:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_4]], 6, [[COPY7]]
+ ; CHECK-NEXT: [[LD1i8_6:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_5]], 7, [[COPY8]]
+ ; CHECK-NEXT: [[LDRBui:%[0-9]+]]:fpr8 = LDRBui [[COPY9]], 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LDRBui]], %subreg.bsub
+ ; CHECK-NEXT: [[LD1i8_7:%[0-9]+]]:fpr128 = LD1i8 [[SUBREG_TO_REG1]], 1, [[COPY10]]
+ ; CHECK-NEXT: [[LD1i8_8:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_7]], 2, [[COPY11]]
+ ; CHECK-NEXT: [[LD1i8_9:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_8]], 3, [[COPY12]]
+ ; CHECK-NEXT: [[LD1i8_10:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_9]], 4, [[COPY13]]
+ ; CHECK-NEXT: [[LD1i8_11:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_10]], 5, [[COPY14]]
+ ; CHECK-NEXT: [[LD1i8_12:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_11]], 6, [[COPY15]]
+ ; CHECK-NEXT: [[LD1i8_13:%[0-9]+]]:fpr128 = LD1i8 [[LD1i8_12]], 7, [[COPY16]]
+ ; CHECK-NEXT: [[ZIP1v2i64_:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD1i8_6]], [[LD1i8_13]]
+ ; CHECK-NEXT: $q0 = COPY [[ZIP1v2i64_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
@@ -250,23 +260,28 @@ body: |
%33:fpr128 = LD1i8 %32, 15, killed %16
$q0 = COPY %33
RET_ReallyLR implicit $q0
-
+...
---
name: negative_pattern_missing_lanes
body: |
bb.0.entry:
liveins: $x0, $x1
+
; CHECK-LABEL: name: negative_pattern_missing_lanes
- ; CHECK: [[LD1:%.*]]:fpr128 = LDRQui $x1, 0
- ; CHECK-NEXT: [[LD2:%.*]]:fpr128 = LD1i32 [[LD1]]
-
+ ; CHECK: liveins: $x0, $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui $x1, 0
+ ; CHECK-NEXT: [[LD1i32_:%[0-9]+]]:fpr128 = LD1i32 [[LDRQui]], 3, [[COPY]]
+ ; CHECK-NEXT: $q0 = COPY [[LD1i32_]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:fpr128 = LDRQui $x1, 0
%2:fpr128 = LD1i32 %1, 3, %0
$q0 = COPY %2
RET_ReallyLR implicit $q0
-
+...
---
name: out_of_order_lanes
body: |
@@ -274,19 +289,21 @@ body: |
liveins: $x0, $x1, $x2, $x3, $x4
; CHECK-LABEL: name: out_of_order_lanes
- ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK: liveins: $x0, $x1, $x2, $x3, $x4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
- ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1
- ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LD_i32]], %subreg.ssub
- ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, [[COPY3]]
- ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY2]], 0
- ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LD1_0]], %subreg.ssub
- ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, [[COPY4]]
- ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]]
- ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+ ; CHECK-NEXT: [[LDRSroX:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LDRSroX]], %subreg.ssub
+ ; CHECK-NEXT: [[LD1i32_:%[0-9]+]]:fpr128 = LD1i32 [[SUBREG_TO_REG]], 1, [[COPY3]]
+ ; CHECK-NEXT: [[LDRSui:%[0-9]+]]:fpr32 = LDRSui [[COPY2]], 0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:fpr128 = SUBREG_TO_REG [[LDRSui]], %subreg.ssub
+ ; CHECK-NEXT: [[LD1i32_1:%[0-9]+]]:fpr128 = LD1i32 [[SUBREG_TO_REG1]], 1, [[COPY4]]
+ ; CHECK-NEXT: [[ZIP1v2i64_:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD1i32_]], [[LD1i32_1]]
+ ; CHECK-NEXT: $q0 = COPY [[ZIP1v2i64_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
@@ -300,7 +317,7 @@ body: |
%9:fpr128 = LD1i32 %8, 3, killed %4
$q0 = COPY %9
RET_ReallyLR implicit $q0
-
+...
---
name: negative_pattern_no_subreg_to_reg
body: |
@@ -308,27 +325,29 @@ body: |
liveins: $x0, $x1, $x2, $x3
; CHECK-LABEL: name: negative_pattern_no_subreg_to_reg
- ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK: liveins: $x0, $x1, $x2, $x3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
- ; CHECK-NEXT: [[INITIAL_VEC:%[0-9]+]]:fpr128 = LDRQui [[COPY]], 0
- ; CHECK-NEXT: [[LD_LANE_1:%[0-9]+]]:fpr128 = LD1i32 [[INITIAL_VEC]], 1, killed [[COPY1]]
- ; CHECK-NEXT: [[LD_LANE_2:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_1]], 2, killed [[COPY2]]
- ; CHECK-NEXT: [[LD_LANE_3:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_2]], 3, killed [[COPY3]]
- ; CHECK-NEXT: $q0 = COPY [[LD_LANE_3]]
+ ; CHECK-NEXT: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[COPY]], 0
+ ; CHECK-NEXT: [[LD1i32_:%[0-9]+]]:fpr128 = LD1i32 [[LDRQui]], 1, killed [[COPY1]]
+ ; CHECK-NEXT: [[LD1i32_1:%[0-9]+]]:fpr128 = LD1i32 [[LD1i32_]], 2, killed [[COPY2]]
+ ; CHECK-NEXT: [[LD1i32_2:%[0-9]+]]:fpr128 = LD1i32 [[LD1i32_1]], 3, killed [[COPY3]]
+ ; CHECK-NEXT: $q0 = COPY [[LD1i32_2]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
%2:gpr64common = COPY $x2
%3:gpr64common = COPY $x3
- %4:fpr128 = LDRQui %0, 0
- %5:fpr128 = LD1i32 %4, 1, killed %1
+ %4:fpr128 = LDRQui %0, 0
+ %5:fpr128 = LD1i32 %4, 1, killed %1
%6:fpr128 = LD1i32 %5, 2, killed %2
%7:fpr128 = LD1i32 %6, 3, killed %3
$q0 = COPY %7
RET_ReallyLR implicit $q0
-
+...
---
name: negative_pattern_multiple_users
body: |
@@ -336,18 +355,20 @@ body: |
liveins: $x0, $x1, $x2, $x3, $x4
; CHECK-LABEL: name: negative_pattern_multiple_users
- ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK: liveins: $x0, $x1, $x2, $x3, $x4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
- ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1
- ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LD_i32]], %subreg.ssub
- ; CHECK-NEXT: [[LD_LANE_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY2]]
- ; CHECK-NEXT: [[LD_LANE_2:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_1]], 2, killed [[COPY3]]
- ; CHECK-NEXT: [[LD_LANE_3:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_2]], 3, killed [[COPY4]]
- ; CHECK-NEXT: $q0 = COPY [[LD_LANE_3]]
- ; CHECK-NEXT: $q1 = COPY [[LD_LANE_2]]
+ ; CHECK-NEXT: [[LDRSroX:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LDRSroX]], %subreg.ssub
+ ; CHECK-NEXT: [[LD1i32_:%[0-9]+]]:fpr128 = LD1i32 [[SUBREG_TO_REG]], 1, killed [[COPY2]]
+ ; CHECK-NEXT: [[LD1i32_1:%[0-9]+]]:fpr128 = LD1i32 [[LD1i32_]], 2, killed [[COPY3]]
+ ; CHECK-NEXT: [[LD1i32_2:%[0-9]+]]:fpr128 = LD1i32 [[LD1i32_1]], 3, killed [[COPY4]]
+ ; CHECK-NEXT: $q0 = COPY [[LD1i32_2]]
+ ; CHECK-NEXT: $q1 = COPY [[LD1i32_1]]
; CHECK-NEXT: RET_ReallyLR implicit $q0, implicit $q1
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
@@ -362,7 +383,7 @@ body: |
$q0 = COPY %9
$q1 = COPY %8
RET_ReallyLR implicit $q0, implicit $q1
-
+...
---
name: aliasing_store_between_vector_loads
alignment: 4
@@ -372,18 +393,20 @@ body: |
liveins: $x0, $x1, $x2, $x3
; CHECK-LABEL: name: aliasing_store_between_vector_loads
- ; CHECK: [[BASE_PTR:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[OFFSET_PTR:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[ALIAS_ADDR:%[0-9]+]]:gpr64common = COPY $x2
- ; CHECK-NEXT: [[OTHER_ADDR:%[0-9]+]]:gpr64common = COPY $x3
- ; CHECK-NEXT: [[LOAD0:%[0-9]+]]:fpr32 = LDRSroX [[BASE_PTR]], killed [[OFFSET_PTR]], 0, 1
- ; CHECK-NEXT: [[VEC0:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LOAD0]], %subreg.ssub
- ; CHECK-NEXT: [[VEC1:%[0-9]+]]:fpr128 = LD1i32 [[VEC0]], 1, [[ALIAS_ADDR]]
- ; CHECK-NEXT: [[CONST:%[0-9]+]]:gpr32 = MOVi32imm 99
- ; CHECK-NEXT: STRWui [[CONST]], [[ALIAS_ADDR]], 0
- ; CHECK-NEXT: [[VEC2:%[0-9]+]]:fpr128 = LD1i32 [[VEC1]], 2, killed [[ALIAS_ADDR]]
- ; CHECK-NEXT: [[VEC3:%[0-9]+]]:fpr128 = LD1i32 [[VEC2]], 3, killed [[OTHER_ADDR]]
- ; CHECK-NEXT: $q0 = COPY [[VEC3]]
+ ; CHECK: liveins: $x0, $x1, $x2, $x3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+ ; CHECK-NEXT: [[LDRSroX:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG killed [[LDRSroX]], %subreg.ssub
+ ; CHECK-NEXT: [[LD1i32_:%[0-9]+]]:fpr128 = LD1i32 [[SUBREG_TO_REG]], 1, [[COPY2]]
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 99
+ ; CHECK-NEXT: STRWui [[MOVi32imm]], [[COPY2]], 0
+ ; CHECK-NEXT: [[LD1i32_1:%[0-9]+]]:fpr128 = LD1i32 [[LD1i32_]], 2, killed [[COPY2]]
+ ; CHECK-NEXT: [[LD1i32_2:%[0-9]+]]:fpr128 = LD1i32 [[LD1i32_1]], 3, killed [[COPY3]]
+ ; CHECK-NEXT: $q0 = COPY [[LD1i32_2]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:gpr64common = COPY $x0
%1:gpr64common = COPY $x1
@@ -398,3 +421,4 @@ body: |
%9:fpr128 = LD1i32 %8, 3, killed %3
$q0 = COPY %9
RET_ReallyLR implicit $q0
+...
>From 73c0f4b2377378957b2134e283b9c631dfcb90d5 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Sat, 24 Oct 2026 08:58:08 +0200
Subject: [PATCH 11/37] [CIR] Narrow blanket -fopenmp NYI guards to OGCG
conditions (#233671)
Ordinary code compiled with -fopenmp hit errorNYI on every assignment,
inc/dec, member access and global reference
---
clang/include/clang/CIR/MissingFeatures.h | 2 +
clang/lib/CIR/CodeGen/CIRGenExpr.cpp | 9 +--
clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp | 6 +-
clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp | 6 +-
clang/lib/CIR/CodeGen/CIRGenModule.cpp | 4 +-
.../CIR/CodeGenOpenMP/non-openmp-code.cpp | 76 +++++++++++++++++++
6 files changed, 88 insertions(+), 15 deletions(-)
create mode 100644 clang/test/CIR/CodeGenOpenMP/non-openmp-code.cpp
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index b7928e0aa38cbe..7beb06541f7b64 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -273,6 +273,8 @@ struct MissingFeatures {
static bool msvcBuiltins() { return false; }
static bool openCL() { return false; }
static bool openMP() { return false; }
+ static bool openMPLastprivateConditional() { return false; }
+ static bool openMPNontemporalDecl() { return false; }
static bool sycl() { return false; }
static bool opTBAA() { return false; }
static bool peepholeProtection() { return false; }
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index a8181c8c6a5b40..2637266cf3feed 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -1906,7 +1906,9 @@ LValue CIRGenFunction::emitMemberExpr(const MemberExpr *e) {
// If the member was explicitly marked as nontemporal, mark it as
// nontemporal. If the base lvalue is marked as nontemporal, mark access
// to children as nontemporal too.
- cgm.errorNYI(e->getSourceRange(), "emitMemberExpr: OpenMP");
+ assert(!cir::MissingFeatures::openMPNontemporalDecl());
+ if (baseLV.isNontemporal())
+ lv.setNontemporal(true);
}
return lv;
}
@@ -2272,10 +2274,7 @@ LValue CIRGenFunction::emitBinaryOperatorLValue(const BinaryOperator *e) {
else
emitStoreThroughLValue(rv, lv);
- if (getLangOpts().OpenMP) {
- cgm.errorNYI(e->getSourceRange(), "openmp");
- return {};
- }
+ assert(!cir::MissingFeatures::openMPLastprivateConditional());
return lv;
}
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index 56f45247e59059..e245287931795c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -1087,8 +1087,7 @@ LValue CIRGenFunction::emitComplexAssignmentLValue(const BinaryOperator *e) {
mlir::Value value; // ignored
LValue lvalue = ComplexExprEmitter(*this).emitBinAssignLValue(e, value);
- if (getLangOpts().OpenMP)
- cgm.errorNYI("emitComplexAssignmentLValue OpenMP");
+ assert(!cir::MissingFeatures::openMPLastprivateConditional());
return lvalue;
}
@@ -1144,8 +1143,7 @@ mlir::Value CIRGenFunction::emitComplexPrePostIncDec(const UnaryOperator *e,
// Store the updated result through the lvalue.
emitStoreOfComplex(loc, incVal, lv, /*isInit=*/false);
- if (getLangOpts().OpenMP)
- cgm.errorNYI(loc, "emitComplexPrePostIncDec OpenMP");
+ assert(!cir::MissingFeatures::openMPLastprivateConditional());
// If this is a postinc, return the value read from memory, otherwise use the
// updated value.
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 5d2b946c9bf420..8e6ded0527cc90 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -634,8 +634,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
mlir::Value emitFixedPointIncDec(const UnaryOperator *e, mlir::Value value,
QualType type);
mlir::Value emitScalarPrePostIncDec(const UnaryOperator *e, LValue lv) {
- if (cgf.getLangOpts().OpenMP)
- cgf.cgm.errorNYI(e->getSourceRange(), "inc/dec OpenMP");
+ assert(!cir::MissingFeatures::openMPLastprivateConditional());
QualType type = e->getSubExpr()->getType();
@@ -1738,8 +1737,7 @@ LValue ScalarExprEmitter::emitCompoundAssignLValue(
else
cgf.emitStoreThroughLValue(RValue::get(result), lhsLV);
- if (cgf.getLangOpts().OpenMP)
- cgf.cgm.errorNYI(e->getSourceRange(), "openmp");
+ assert(!cir::MissingFeatures::openMPLastprivateConditional());
return lhsLV;
}
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index f91074afe6f18b..81b8c59ef90dc6 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -1033,8 +1033,8 @@ LangAS CIRGenModule::getGlobalVarAddressSpace(const VarDecl *d) {
return LangAS::cuda_device;
}
- if (langOpts.OpenMP)
- errorNYI("OpenMP global address space");
+ if (langOpts.OpenMP && d && d->hasAttr<OMPAllocateDeclAttr>())
+ errorNYI(d->getSourceRange(), "OpenMP global address space");
return getTargetCIRGenInfo().getGlobalVarAddressSpace(*this, d);
}
diff --git a/clang/test/CIR/CodeGenOpenMP/non-openmp-code.cpp b/clang/test/CIR/CodeGenOpenMP/non-openmp-code.cpp
new file mode 100644
index 00000000000000..d3f4e62b3d10dd
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenMP/non-openmp-code.cpp
@@ -0,0 +1,76 @@
+// RUN: %clang_cc1 -fopenmp -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t-offload.cir
+// RUN: FileCheck --input-file=%t-offload.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -fopenmp -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefixes=LLVM,LLVMCIR
+// RUN: %clang_cc1 -fopenmp -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefixes=LLVM,OGCG
+
+// Ordinary code compiled with -fopenmp.
+
+struct S {
+ int x;
+};
+
+void scalar_ops(S *s, int v) {
+ static int g;
+ g = v;
+ g += v;
+ ++g;
+ s->x = g;
+}
+
+// CIR: cir.global "private" internal dso_local @[[G:_ZZ10scalar_opsP1SiE1g]] = #cir.int<0> : !s32i
+// CIR: cir.func{{.*}} @_Z10scalar_opsP1Si
+// CIR: %[[G_PTR:.*]] = cir.get_global @[[G]] : !cir.ptr<!s32i>
+// CIR: cir.store{{.*}} %{{.*}}, %[[G_PTR]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[G1:.*]] = cir.load{{.*}} %[[G_PTR]] : !cir.ptr<!s32i>, !s32i
+// CIR: %[[ADD:.*]] = cir.add nsw %[[G1]], %{{.*}} : !s32i
+// CIR: cir.store{{.*}} %[[ADD]], %[[G_PTR]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[G2:.*]] = cir.load{{.*}} %[[G_PTR]] : !cir.ptr<!s32i>, !s32i
+// CIR: %[[INC:.*]] = cir.inc nsw %[[G2]] : !s32i
+// CIR: cir.store{{.*}} %[[INC]], %[[G_PTR]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[X:.*]] = cir.get_member %{{.*}}[0] {name = "x"} : !cir.ptr<!rec_S> -> !cir.ptr<!s32i>
+// CIR: cir.store{{.*}} %{{.*}}, %[[X]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: @[[G:_ZZ10scalar_opsP1SiE1g]] = internal global i32 0
+// LLVM: define{{.*}} void @_Z10scalar_opsP1Si
+// LLVM: store i32 %{{.*}}, ptr @[[G]]
+// LLVM: %[[G1:.*]] = load i32, ptr @[[G]]
+// LLVM: %[[ADD:.*]] = add nsw i32 %[[G1]], %{{.*}}
+// LLVM: store i32 %[[ADD]], ptr @[[G]]
+// LLVM: %[[G2:.*]] = load i32, ptr @[[G]]
+// LLVM: %[[INC:.*]] = add nsw i32 %[[G2]], 1
+// LLVM: store i32 %[[INC]], ptr @[[G]]
+// LLVM: %[[X:.*]] = getelementptr inbounds nuw %struct.S, ptr %{{.*}}, i32 0, i32 0
+// LLVM: store i32 %{{.*}}, ptr %[[X]]
+
+void complex_ops(float _Complex *c, float _Complex v) {
+ *c = v;
+ ++*c;
+}
+
+// CIR: cir.func{{.*}} @_Z11complex_opsPCfS_
+// CIR: %[[V:.*]] = cir.load align(4) %{{.*}} : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[C_PTR:.*]] = cir.load deref{{.*}} %{{.*}} : !cir.ptr<!cir.ptr<!cir.complex<!cir.float>>>, !cir.ptr<!cir.complex<!cir.float>>
+// CIR: cir.store{{.*}} %[[V]], %[[C_PTR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+// CIR: %[[C_PTR2:.*]] = cir.load deref{{.*}} %{{.*}} : !cir.ptr<!cir.ptr<!cir.complex<!cir.float>>>, !cir.ptr<!cir.complex<!cir.float>>
+// CIR: %[[C:.*]] = cir.load{{.*}} %[[C_PTR2]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[REAL:.*]] = cir.complex.real %[[C]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[IMAG:.*]] = cir.complex.imag %[[C]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[INC:.*]] = cir.fadd %[[REAL]], %{{.*}} : !cir.float
+// CIR: %[[RES:.*]] = cir.complex.create %[[INC]], %[[IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[RES]], %{{.*}} : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVMCIR: define{{.*}} void @_Z11complex_opsPCfS_
+// LLVMCIR: store { float, float } %{{.*}}, ptr %{{.*}}
+// LLVMCIR: %[[REAL:.*]] = extractvalue { float, float } %{{.*}}, 0
+// LLVMCIR: %[[INC:.*]] = fadd float %[[REAL]], 1.000000e+00
+// LLVMCIR: store { float, float } %{{.*}}, ptr %{{.*}}
+
+// OGCG: define{{.*}} void @_Z11complex_opsPCfS_
+// OGCG: store float %{{.*}}, ptr %{{.*}}
+// OGCG: store float %{{.*}}, ptr %{{.*}}
+// OGCG: %[[REAL:.*]] = load float, ptr %{{.*}}
+// OGCG: %[[INC:.*]] = fadd float %[[REAL]], 1.000000e+00
>From 9c20e5ef1410a701e93fcadfe0a305af3fdd63fe Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Sat, 24 Oct 2026 08:58:24 +0200
Subject: [PATCH 12/37] [CIR] Fix cir.cast fold crash on a same type integral
cast (#234236)
The fold called fold() on the source defining op, which is null for
block arguments
---
clang/lib/CIR/Dialect/IR/CIRDialect.cpp | 8 +-------
clang/test/CIR/Transforms/canonicalize.cir | 7 +++++++
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 838ea1e953e9b9..84e7611335b534 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -1295,13 +1295,7 @@ OpFoldResult cir::CastOp::fold(FoldAdaptor adaptor) {
if (getSrc().getType() == getType()) {
switch (getKind()) {
- case cir::CastKind::integral: {
- llvm::SmallVector<mlir::OpFoldResult, 1> foldResults;
- auto foldOrder = getSrc().getDefiningOp()->fold(foldResults);
- if (foldOrder.succeeded() && mlir::isa<mlir::Attribute>(foldResults[0]))
- return mlir::cast<mlir::Attribute>(foldResults[0]);
- return {};
- }
+ case cir::CastKind::integral:
case cir::CastKind::bitcast:
case cir::CastKind::address_space:
case cir::CastKind::float_complex:
diff --git a/clang/test/CIR/Transforms/canonicalize.cir b/clang/test/CIR/Transforms/canonicalize.cir
index c5b72f424aa643..a65df32833f196 100644
--- a/clang/test/CIR/Transforms/canonicalize.cir
+++ b/clang/test/CIR/Transforms/canonicalize.cir
@@ -216,6 +216,13 @@ module {
// CHECK-NEXT: %[[CAST3:.*]] = cir.cast integral %[[CAST2]] : !s32i -> !s64i
// CHECK-NEXT: cir.return %[[CAST3]] : !s64i
+ cir.func @cast_integral_same_type(%arg0: !s32i) -> !s32i {
+ %0 = cir.cast integral %arg0 : !s32i -> !s32i
+ cir.return %0 : !s32i
+ }
+ // CHECK: cir.func{{.*}} @cast_integral_same_type(%[[ARG0:.*]]: !s32i) -> !s32i
+ // CHECK-NEXT: cir.return %[[ARG0]] : !s32i
+
cir.func @cast_poison() -> !s64i {
%0 = cir.const #cir.poison : !s32i
%1 = cir.cast integral %0 : !s32i -> !s64i
>From b968e745fb9552b0da3b7fdfaf70c2c31eb7bece Mon Sep 17 00:00:00 2001
From: "Michael G. Kazakov" <mike.kazakov at gmail.com>
Date: Sat, 24 Oct 2026 08:29:01 +0100
Subject: [PATCH 13/37] [libc++] Add benchmarks for std::reduce and
std::transform_reduce (#233374)
The PR adds benchmarks for these algorithms:
- `std::reduce()`
- `std::transform_reduce()`, unary transform
- `std::transform_reduce()`, binary transform
The benchmarks run minimal reduction and transformation operators with
`int`, `float` and `double` parameters.
`std::vector` and `std::list` containers are tested, `std::deque` is not
included to reduce the number of added benchmarks.
Fixes #226211
---------
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
.../test/benchmarks/numeric/reduce.bench.cpp | 80 +++++++++++
.../numeric/transform_reduce.bench.cpp | 127 ++++++++++++++++++
2 files changed, 207 insertions(+)
create mode 100644 libcxx/test/benchmarks/numeric/reduce.bench.cpp
create mode 100644 libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
diff --git a/libcxx/test/benchmarks/numeric/reduce.bench.cpp b/libcxx/test/benchmarks/numeric/reduce.bench.cpp
new file mode 100644
index 00000000000000..ece7578d18ff32
--- /dev/null
+++ b/libcxx/test/benchmarks/numeric/reduce.bench.cpp
@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+
+#include <algorithm>
+#include <cstddef>
+#include <iterator>
+#include <list>
+#include <numeric>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include "../GenerateInput.h"
+#include "test_macros.h"
+
+// Keep values small so that signed integer multiplication doesn't overflow
+template <class T>
+T random_value() {
+ if constexpr (std::is_integral_v<T>)
+ return getRandomInteger<T>(1, 1000);
+ else
+ return getRandomFloat<T>(1, 1000);
+}
+
+int main(int argc, char** argv) {
+ // std::reduce(first, last, init, reduce)
+ {
+ auto bm = []<class Container>(std::string name) {
+ benchmark::RegisterBenchmark(
+ name,
+ [](auto& st) TEST_ALIGN_BENCHMARK {
+ std::size_t const size = st.range(0);
+ using ValueType = typename Container::value_type;
+
+ Container c;
+ std::generate_n(std::back_inserter(c), size, [] { return random_value<ValueType>(); });
+ ValueType init = static_cast<ValueType>(1);
+
+ auto reduce = [](auto x, auto y) {
+ if constexpr (std::is_floating_point_v<decltype(x)>) {
+ // Compilers aren't allowed to optimize away the FP divisions and multiplications without fast-math.
+ return x / 3. * 3. / 5. * 5. + y / 7. * 7. / 9. * 9.;
+ } else {
+ // p=46337 is the largest prime with '(p-1)^2 <= INT_MAX', so the chain of reductions never overflows.
+ constexpr decltype(x) p = 46337;
+ return x * y % p;
+ }
+ };
+ for (auto _ : st) {
+ benchmark::ClobberMemory();
+ auto result = std::reduce(c.begin(), c.end(), init, reduce);
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(32)
+ ->Arg(50) // non power-of-two
+ ->Arg(1024)
+ ->Arg(8192);
+ };
+ bm.operator()<std::vector<int>>("std::reduce(vector<int>)");
+ bm.operator()<std::vector<float>>("std::reduce(vector<float>)");
+ bm.operator()<std::vector<double>>("std::reduce(vector<double>)");
+ bm.operator()<std::list<int>>("std::reduce(list<int>)");
+ bm.operator()<std::list<float>>("std::reduce(list<float>)");
+ bm.operator()<std::list<double>>("std::reduce(list<double>)");
+ }
+
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+ benchmark::Shutdown();
+ return 0;
+}
diff --git a/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp b/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
new file mode 100644
index 00000000000000..192308f1a9031d
--- /dev/null
+++ b/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
@@ -0,0 +1,127 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+
+#include <algorithm>
+#include <cstddef>
+#include <iterator>
+#include <list>
+#include <numeric>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include "../GenerateInput.h"
+#include "test_macros.h"
+
+// Keep values small so that signed integer multiplication doesn't overflow
+template <class T>
+T random_value() {
+ if constexpr (std::is_integral_v<T>)
+ return getRandomInteger<T>(1, 1000);
+ else
+ return getRandomFloat<T>(1, 1000);
+}
+
+int main(int argc, char** argv) {
+ // std::transform_reduce(first, last, init, reduce, transform)
+ {
+ auto bm = []<class Container>(std::string name) {
+ benchmark::RegisterBenchmark(
+ name,
+ [](auto& st) TEST_ALIGN_BENCHMARK {
+ std::size_t const size = st.range(0);
+ using ValueType = typename Container::value_type;
+
+ Container c;
+ std::generate_n(std::back_inserter(c), size, [] { return random_value<ValueType>(); });
+ ValueType init = static_cast<ValueType>(1);
+
+ auto transform = [](auto x) { return x * 3; };
+ auto reduce = [](auto x, auto y) {
+ if constexpr (std::is_floating_point_v<decltype(x)>) {
+ // Compilers aren't allowed to optimize away the FP divisions and multiplications without fast-math.
+ return x / 3. * 3. / 5. * 5. + y / 7. * 7. / 9. * 9.;
+ } else {
+ // p=46337 is the largest prime with '(p-1)^2 <= INT_MAX', so the chain of reductions never overflows.
+ constexpr decltype(x) p = 46337;
+ return x * y % p;
+ }
+ };
+
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::ClobberMemory();
+ auto result = std::transform_reduce(c.begin(), c.end(), init, reduce, transform);
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(32)
+ ->Arg(50) // non power-of-two
+ ->Arg(1024)
+ ->Arg(8192);
+ };
+ bm.operator()<std::vector<int>>("std::transform_reduce(vector<int>)");
+ bm.operator()<std::vector<float>>("std::transform_reduce(vector<float>)");
+ bm.operator()<std::vector<double>>("std::transform_reduce(vector<double>)");
+ bm.operator()<std::list<int>>("std::transform_reduce(list<int>)");
+ bm.operator()<std::list<float>>("std::transform_reduce(list<float>)");
+ bm.operator()<std::list<double>>("std::transform_reduce(list<double>)");
+ }
+
+ // std::transform_reduce(first1, last1, first2, init, reduce, transform)
+ {
+ auto bm = []<class Container>(std::string name) {
+ benchmark::RegisterBenchmark(
+ name,
+ [](auto& st) TEST_ALIGN_BENCHMARK {
+ std::size_t const size = st.range(0);
+ using ValueType = typename Container::value_type;
+
+ Container c1, c2;
+ std::generate_n(std::back_inserter(c1), size, [] { return random_value<ValueType>(); });
+ std::generate_n(std::back_inserter(c2), size, [] { return random_value<ValueType>(); });
+ ValueType init = static_cast<ValueType>(1);
+
+ auto transform = [](auto x, auto y) { return x - y; };
+ auto reduce = [](auto x, auto y) {
+ if constexpr (std::is_floating_point_v<decltype(x)>) {
+ // Compilers aren't allowed to optimize away the FP divisions and multiplications without fast-math.
+ return x / 3. * 3. / 5. * 5. + y / 7. * 7. / 9. * 9.;
+ } else {
+ // p=46337 is the largest prime with '(p-1)^2 <= INT_MAX', so the chain of reductions never overflows.
+ constexpr decltype(x) p = 46337;
+ return x * y % p;
+ }
+ };
+
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::ClobberMemory();
+ auto result = std::transform_reduce(c1.begin(), c1.end(), c2.begin(), init, reduce, transform);
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(32)
+ ->Arg(50) // non power-of-two
+ ->Arg(1024)
+ ->Arg(8192);
+ };
+ bm.operator()<std::vector<int>>("std::transform_reduce(vector<int>, vector<int>)");
+ bm.operator()<std::vector<float>>("std::transform_reduce(vector<float>, vector<float>)");
+ bm.operator()<std::vector<double>>("std::transform_reduce(vector<double>, vector<double>)");
+ bm.operator()<std::list<int>>("std::transform_reduce(list<int>, list<int>)");
+ bm.operator()<std::list<float>>("std::transform_reduce(list<float>, list<float>)");
+ bm.operator()<std::list<double>>("std::transform_reduce(list<double>, list<double>)");
+ }
+
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+ benchmark::Shutdown();
+ return 0;
+}
>From c467c6b1e53dc0939ab78a644eceea6e61cf4953 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder at redhat.com>
Date: Sat, 24 Oct 2026 09:30:28 +0200
Subject: [PATCH 14/37] [clang][ExprConst] Add missing `FastEvaluateAsRValue`
in `EvaluateKnownConstInt{,CheckOverflow}` (#234552)
This was missing for some reason.
---
clang/lib/AST/ExprConstant.cpp | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index b37aa4de5f2b10..3762bc83c779b0 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -22802,6 +22802,13 @@ APSInt Expr::EvaluateKnownConstInt(const ASTContext &Ctx) const {
ExprTimeTraceScope TimeScope(this, Ctx, "EvaluateKnownConstInt");
EvalResult EVResult;
+ bool IsConst;
+ if (FastEvaluateAsRValue(this, EVResult.Val, Ctx, IsConst)) {
+ assert(IsConst);
+ assert(EVResult.Val.isInt() && "Expression did not evaluate to integer");
+ return EVResult.Val.getInt();
+ }
+
if (Ctx.getLangOpts().EnableNewConstInterp) {
interp::EvalSettings Settings(EvaluationMode::IgnoreSideEffects, EVResult,
/*SProxy=*/nullptr);
@@ -22835,6 +22842,13 @@ APSInt Expr::EvaluateKnownConstIntCheckOverflow(
EvalResult EVResult;
EVResult.Diag = Diag;
+ bool IsConst;
+ if (FastEvaluateAsRValue(this, EVResult.Val, Ctx, IsConst)) {
+ assert(IsConst);
+ assert(EVResult.Val.isInt() && "Expression did not evaluate to integer");
+ return EVResult.Val.getInt();
+ }
+
if (Ctx.getLangOpts().EnableNewConstInterp) {
interp::EvalSettings Settings(EvaluationMode::IgnoreSideEffects, EVResult,
/*SProxy=*/nullptr);
>From ee8cfbf7c22ba7da178b9bef0308fc3ff3322b5d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sat, 24 Oct 2026 00:40:04 -0700
Subject: [PATCH 15/37] AMDGPU: Use S_ADD_I32 to select uniform uaddsat
(#235242)
95ff1fa00ed4 selected uniform uaddsat.i32 to an S_ADD_U32 sequence.
S_ADD_U32 defines a carry out, so when the instructions have to be
moved to the VALU it is replaced with V_ADD_CO_U32_e32, which doesn't
exist on gfx10+. This would later assert when encoded. moveToVALU
should probably learn to handle that correctly, or at least assert.
The carry isn't needed, so use S_ADD_I32 like the usubsat pattern,
which is moved to V_ADD_U32_e64 where available.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SOPInstructions.td | 2 +-
llvm/test/CodeGen/AMDGPU/add-max.ll | 2 +-
llvm/test/CodeGen/AMDGPU/uaddsat.ll | 87 +++++++++++++++++++++--
3 files changed, 85 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index b336af3bcd01f8..8ad7c3744e96e9 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -2089,7 +2089,7 @@ let AddedComplexity = 20 in {
let AddedComplexity = 20 in {
def : GCNPat<
(i32 (UniformBinFrag<uaddsat> i32:$src0, i32:$src1)),
- (S_ADD_U32 (S_MIN_U32 $src0, (S_NOT_B32 $src1)), $src1)
+ (S_ADD_I32 (S_MIN_U32 $src0, (S_NOT_B32 $src1)), $src1)
>;
def : GCNPat<
(i32 (UniformBinFrag<usubsat> i32:$src0, i32:$src1)),
diff --git a/llvm/test/CodeGen/AMDGPU/add-max.ll b/llvm/test/CodeGen/AMDGPU/add-max.ll
index 483a70ffbcd8c5..587a95ebf67fc2 100644
--- a/llvm/test/CodeGen/AMDGPU/add-max.ll
+++ b/llvm/test/CodeGen/AMDGPU/add-max.ll
@@ -57,7 +57,7 @@ define amdgpu_ps float @add_max_u32_sss(i32 inreg %a, i32 inreg %b, i32 inreg %c
; SDAG-NEXT: s_not_b32 s3, s1
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; SDAG-NEXT: s_min_u32 s0, s0, s3
-; SDAG-NEXT: s_add_co_u32 s0, s0, s1
+; SDAG-NEXT: s_add_co_i32 s0, s0, s1
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; SDAG-NEXT: s_max_u32 s0, s0, s2
; SDAG-NEXT: v_mov_b32_e32 v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
index a087c00ad49bf7..ac413d8a6e837d 100644
--- a/llvm/test/CodeGen/AMDGPU/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
@@ -363,7 +363,7 @@ define i32 @s_uaddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_not_b32 s4, s17
; GFX8-NEXT: s_min_u32 s4, s16, s4
-; GFX8-NEXT: s_add_u32 s4, s4, s17
+; GFX8-NEXT: s_add_i32 s4, s4, s17
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -372,7 +372,7 @@ define i32 @s_uaddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_not_b32 s4, s17
; GFX9-NEXT: s_min_u32 s4, s16, s4
-; GFX9-NEXT: s_add_u32 s4, s4, s17
+; GFX9-NEXT: s_add_i32 s4, s4, s17
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -381,7 +381,7 @@ define i32 @s_uaddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_not_b32 s4, s17
; GFX10-NEXT: s_min_u32 s4, s16, s4
-; GFX10-NEXT: s_add_u32 s4, s4, s17
+; GFX10-NEXT: s_add_i32 s4, s4, s17
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -391,7 +391,7 @@ define i32 @s_uaddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
; GFX11-NEXT: s_not_b32 s2, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_min_u32 s0, s0, s2
-; GFX11-NEXT: s_add_u32 s0, s0, s1
+; GFX11-NEXT: s_add_i32 s0, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -399,6 +399,85 @@ define i32 @s_uaddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
ret i32 %result
}
+; The uniform uaddsat is selected to SALU instructions, which are moved to the
+; VALU because the result of the load is in a VGPR.
+define amdgpu_ps void @s_uaddsat_i32_vgpr_operand(i32 inreg %lhs, ptr addrspace(8) inreg %rsrc) {
+; GFX6-LABEL: s_uaddsat_i32_vgpr_operand:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 s7, s4
+; GFX6-NEXT: s_mov_b32 s6, s3
+; GFX6-NEXT: s_mov_b32 s5, s2
+; GFX6-NEXT: s_mov_b32 s4, s1
+; GFX6-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_not_b32_e32 v1, v0
+; GFX6-NEXT: v_min_u32_e32 v1, s0, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:4
+; GFX6-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_uaddsat_i32_vgpr_operand:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_mov_b32 s7, s4
+; GFX8-NEXT: s_mov_b32 s6, s3
+; GFX8-NEXT: s_mov_b32 s5, s2
+; GFX8-NEXT: s_mov_b32 s4, s1
+; GFX8-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_not_b32_e32 v1, v0
+; GFX8-NEXT: v_min_u32_e32 v1, s0, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v1, v0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:4
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_uaddsat_i32_vgpr_operand:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_mov_b32 s7, s4
+; GFX9-NEXT: s_mov_b32 s6, s3
+; GFX9-NEXT: s_mov_b32 s5, s2
+; GFX9-NEXT: s_mov_b32 s4, s1
+; GFX9-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_not_b32_e32 v1, v0
+; GFX9-NEXT: v_min_u32_e32 v1, s0, v1
+; GFX9-NEXT: v_add_u32_e32 v0, v1, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:4
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: s_uaddsat_i32_vgpr_operand:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_mov_b32 s7, s4
+; GFX10-NEXT: s_mov_b32 s6, s3
+; GFX10-NEXT: s_mov_b32 s5, s2
+; GFX10-NEXT: s_mov_b32 s4, s1
+; GFX10-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_not_b32_e32 v1, v0
+; GFX10-NEXT: v_min_u32_e32 v1, s0, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v0, v1, v0
+; GFX10-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:4
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_uaddsat_i32_vgpr_operand:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_mov_b32 s7, s4
+; GFX11-NEXT: s_mov_b32 s6, s3
+; GFX11-NEXT: s_mov_b32 s5, s2
+; GFX11-NEXT: s_mov_b32 s4, s1
+; GFX11-NEXT: buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_not_b32_e32 v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_u32_e32 v1, s0, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v0, v1, v0
+; GFX11-NEXT: buffer_store_b32 v0, off, s[4:7], 0 offset:4
+; GFX11-NEXT: s_endpgm
+ %rhs = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
+ call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %result, ptr addrspace(8) %rsrc, i32 4, i32 0, i32 0)
+ ret void
+}
+
define i32 @v_uaddsat_i32(i32 %lhs, i32 %rhs) {
; GFX6-LABEL: v_uaddsat_i32:
; GFX6: ; %bb.0:
>From 612b8b1661a48ff9db405e3054925837104c10b0 Mon Sep 17 00:00:00 2001
From: lijinpei-amd <jinpli at amd.com>
Date: Sat, 24 Oct 2026 16:34:25 +0800
Subject: [PATCH 16/37] [AMDGPU] Keep dst flags when unpacking packed F32
instructions (#234858)
performF32Unpacking copied the new instructions' dst operands by value,
so
its setIsUndef and setIsRenamable calls never reached the instructions.
Take
the operands by reference.
Assisted-by: Claude Opus 5.5
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 4 +-
.../si-pre-emit-peephole-unpack-dst-flags.mir | 20 +++++
...ck-non-coissue-insts-post-ra-scheduler.mir | 88 +++++++++----------
3 files changed, 66 insertions(+), 46 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-unpack-dst-flags.mir
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 89461de967b85d..b8fe4f8587ed90 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -776,13 +776,13 @@ void SIPreEmitPeephole::performF32Unpacking(MachineInstr &I) {
MachineInstrBuilder Op0LOp1L =
createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/false);
- MachineOperand LoDstOp = Op0LOp1L->getOperand(0);
+ MachineOperand &LoDstOp = Op0LOp1L->getOperand(0);
LoDstOp.setIsUndef(DstOp.isUndef());
MachineInstrBuilder Op0HOp1H =
createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/true);
- MachineOperand HiDstOp = Op0HOp1H->getOperand(0);
+ MachineOperand &HiDstOp = Op0HOp1H->getOperand(0);
uint32_t IFlags = I.getFlags();
Op0LOp1L->setFlags(IFlags);
diff --git a/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-unpack-dst-flags.mir b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-unpack-dst-flags.mir
new file mode 100644
index 00000000000000..f3a917ae3b00e1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-unpack-dst-flags.mir
@@ -0,0 +1,20 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=si-pre-emit-peephole -o - %s | FileCheck %s
+
+---
+name: unpack_keeps_renamable
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr8_vgpr9
+ ; CHECK-LABEL: name: unpack_keeps_renamable
+ ; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr8_vgpr9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16BF16_1K_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr10 = nofpexcept V_FMA_F32_e64 0, $vgpr8, 0, $vgpr8, 0, $vgpr8, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr11 = nofpexcept V_FMA_F32_e64 0, $vgpr9, 0, $vgpr9, 0, $vgpr9, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16BF16_1K_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr10_vgpr11 = nofpexcept V_PK_FMA_F32 8, $vgpr8_vgpr9, 8, $vgpr8_vgpr9, 8, $vgpr8_vgpr9, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
index 5685df559b0828..7cd6f57e0036c1 100644
--- a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
+++ b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
@@ -31,8 +31,8 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_pk_mul_unpacking_f32
@@ -53,8 +53,8 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_pk_mul_unpacking_f32
@@ -124,8 +124,8 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_op_sel_selection_unpacking_f32
@@ -146,8 +146,8 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_op_sel_selection_unpacking_f32
@@ -217,8 +217,8 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_op_sel_hi_selection_unpacking_f32
@@ -239,8 +239,8 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_op_sel_hi_selection_unpacking_f32
@@ -322,12 +322,12 @@ body: |
; GFX950-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr18, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr19, implicit $exec, implicit $exec
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_BF8_BF8_e64 killed $vgpr4_vgpr5, killed $vgpr2_vgpr3, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 1, 2, 3, implicit $mode, implicit $exec, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr2 = nofpexcept V_ADD_F32_e64 0, killed $sgpr2, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr3 = nofpexcept V_ADD_F32_e64 0, killed $sgpr3, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr6 = nofpexcept V_ADD_F32_e64 0, killed $sgpr6, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr7 = nofpexcept V_ADD_F32_e64 0, killed $sgpr7, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr4 = nofpexcept V_ADD_F32_e64 0, killed $sgpr4, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr5 = nofpexcept V_ADD_F32_e64 0, killed $sgpr5, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr2 = nofpexcept V_ADD_F32_e64 0, killed $sgpr2, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr3 = nofpexcept V_ADD_F32_e64 0, killed $sgpr3, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr6 = nofpexcept V_ADD_F32_e64 0, killed $sgpr6, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr7 = nofpexcept V_ADD_F32_e64 0, killed $sgpr7, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr4 = nofpexcept V_ADD_F32_e64 0, killed $sgpr4, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr5 = nofpexcept V_ADD_F32_e64 0, killed $sgpr5, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: renamable $vgpr10_vgpr11 = nofpexcept V_PK_ADD_F32 8, killed $sgpr10_sgpr11, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
@@ -361,12 +361,12 @@ body: |
; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr18, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr19, implicit $exec, implicit $exec
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_BF8_BF8_e64 killed $vgpr4_vgpr5, killed $vgpr2_vgpr3, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 1, 2, 3, implicit $mode, implicit $exec, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr2 = nofpexcept V_ADD_F32_e64 0, killed $sgpr2, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr3 = nofpexcept V_ADD_F32_e64 0, killed $sgpr3, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr6 = nofpexcept V_ADD_F32_e64 0, killed $sgpr6, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr7 = nofpexcept V_ADD_F32_e64 0, killed $sgpr7, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr4 = nofpexcept V_ADD_F32_e64 0, killed $sgpr4, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr5 = nofpexcept V_ADD_F32_e64 0, killed $sgpr5, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr2 = nofpexcept V_ADD_F32_e64 0, killed $sgpr2, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr3 = nofpexcept V_ADD_F32_e64 0, killed $sgpr3, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr6 = nofpexcept V_ADD_F32_e64 0, killed $sgpr6, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr7 = nofpexcept V_ADD_F32_e64 0, killed $sgpr7, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr4 = nofpexcept V_ADD_F32_e64 0, killed $sgpr4, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr5 = nofpexcept V_ADD_F32_e64 0, killed $sgpr5, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: renamable $vgpr10_vgpr11 = nofpexcept V_PK_ADD_F32 8, killed $sgpr10_sgpr11, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
@@ -465,8 +465,8 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_pk_fma_unpacking_f32
@@ -485,8 +485,8 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_pk_fma_unpacking_f32
@@ -550,8 +550,8 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr4 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr5 = nofpexcept V_MUL_F32_e64 0, $sgpr31, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr4 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr5 = nofpexcept V_MUL_F32_e64 0, $sgpr31, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_FMA_F32 8, killed $sgpr30_sgpr31, 8, $vgpr4_vgpr5, 0, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
@@ -571,8 +571,8 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr4 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr5 = nofpexcept V_MUL_F32_e64 0, $sgpr31, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr4 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr5 = nofpexcept V_MUL_F32_e64 0, $sgpr31, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_FMA_F32 8, killed $sgpr30_sgpr31, 8, $vgpr4_vgpr5, 0, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
@@ -740,8 +740,8 @@ body: |
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 $sgpr15, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr6 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr7 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, $vgpr5, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, $vgpr5, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_opsel_register_is_correctly_marked_as_killed
@@ -762,8 +762,8 @@ body: |
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 $sgpr15, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr6 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr7 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, $vgpr5, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, $vgpr5, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_opsel_register_is_correctly_marked_as_killed
@@ -1011,8 +1011,8 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_unpacking_with_imm_input
@@ -1033,8 +1033,8 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_unpacking_with_imm_input
@@ -1104,8 +1104,8 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 1, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 1, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 1, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 1, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_neg_lo_hi_post_unpacking
@@ -1126,8 +1126,8 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 1, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 1, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 1, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 1, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_neg_lo_hi_post_unpacking
>From 7177619fd2ca2a468a8725fc26c3264ea2499095 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Sat, 24 Oct 2026 10:59:08 +0200
Subject: [PATCH 17/37] [OpenMP][offload] Add NumComputeUnits and Backend to
GeometryProps (#234973)
Co-authored-by: Alex Duran <alejandro.duran at intel.com>
---
offload/include/device.h | 17 +++++++++++---
offload/libompaccsupport/PluginManager.cpp | 10 ++-------
offload/libompaccsupport/device.cpp | 26 +++++++++++++++++++---
3 files changed, 39 insertions(+), 14 deletions(-)
diff --git a/offload/include/device.h b/offload/include/device.h
index 11dc6931a97cc9..fb69e673998f97 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -101,19 +101,24 @@ struct DeviceGeometryPropsTy {
uint32_t DefaultNumBlocks = 0;
/// Maximum amount of shared memory, in bytes, a block can use in a kernel.
uint64_t MaxBlockSharedMemSize = 0;
+ /// Number of compute units, or zero if the device does not report it.
+ uint32_t NumComputeUnits = 0;
+ /// Backend of the platform of the device.
+ ol_platform_backend_t Backend = OL_PLATFORM_BACKEND_UNKNOWN;
};
struct PluginManager;
struct DeviceTy {
int32_t DeviceID;
- GenericPluginTy *RTL;
- int32_t RTLDeviceID;
+ GenericPluginTy *RTL = nullptr;
+ int32_t RTLDeviceID = -1;
+ ol_platform_handle_t Platform;
ol_device_handle_t DeviceHandle;
PluginManager *PM;
ol_context_handle_t Context = nullptr;
- DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID,
+ DeviceTy(int32_t DeviceID, ol_platform_handle_t Platform,
ol_device_handle_t DeviceHandle, PluginManager *PM);
// DeviceTy is not copyable
DeviceTy(const DeviceTy &D) = delete;
@@ -133,6 +138,9 @@ struct DeviceTy {
/// Whether this device executes kernels on the host CPU.
bool isHostDevice() const { return IsHostDevice; }
+ /// Get this device's backend.
+ ol_platform_backend_t getBackend() const { return Backend; }
+
/// Load \p Img onto the device and return the resulting program.
llvm::Expected<ProgramTy> loadBinary(__tgt_device_image *Img);
@@ -373,6 +381,9 @@ struct DeviceTy {
/// Whether this is a host device.
bool IsHostDevice = false;
+ /// The backend of this device.
+ ol_platform_backend_t Backend = OL_PLATFORM_BACKEND_UNKNOWN;
+
/// Handler to collect and organize host-2-device mapping information.
MappingInfoTy MappingInfo;
diff --git a/offload/libompaccsupport/PluginManager.cpp b/offload/libompaccsupport/PluginManager.cpp
index 4cab03e40f00cd..b49ace3404443d 100644
--- a/offload/libompaccsupport/PluginManager.cpp
+++ b/offload/libompaccsupport/PluginManager.cpp
@@ -39,7 +39,6 @@ using namespace llvm::omp::target::helpers;
using llvm::omp::target::plugin::GenericKernelTy;
namespace llvm::offload::tmp {
GenericPluginTy *__ol_tgt_GetPluginFromPlatform(ol_platform_handle_t Platform);
-int32_t __ol_tgt_GetPluginDeviceId(ol_device_handle_t Device);
GenericKernelTy *__ol_tgt_GetKernelFromSymbol(ol_symbol_handle_t Symbol);
} // namespace llvm::offload::tmp
@@ -104,21 +103,16 @@ bool PluginManager::initializeDevice(ol_device_handle_t DeviceHandle) {
return false;
}
- GenericPluginTy &Plugin =
- *llvm::offload::tmp::__ol_tgt_GetPluginFromPlatform(PlatformHandle);
- int32_t DeviceId =
- llvm::offload::tmp::__ol_tgt_GetPluginDeviceId(DeviceHandle);
-
// Initialize the device information for the RTL we are about to use.
auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
int32_t UserId = ExclusiveDevicesAccessor->size();
auto Device =
- std::make_unique<DeviceTy>(&Plugin, UserId, DeviceId, DeviceHandle, this);
+ std::make_unique<DeviceTy>(UserId, PlatformHandle, DeviceHandle, this);
if (auto Err = Device->init()) {
std::string InfoMsg = toString(std::move(Err));
- ODBG(ODT_Init) << "Failed to init device " << DeviceId << ": " << InfoMsg;
+ ODBG(ODT_Init) << "Failed to init device " << UserId << ": " << InfoMsg;
return false;
}
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 0bbfc9ce8698f9..99b7b1a074dff8 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -45,6 +45,11 @@ using namespace llvm::omp::target::error;
using namespace llvm::omp::target::plugin;
using namespace llvm::omp::target::debug;
+namespace llvm::offload::tmp {
+GenericPluginTy *__ol_tgt_GetPluginFromPlatform(ol_platform_handle_t Platform);
+int32_t __ol_tgt_GetPluginDeviceId(ol_device_handle_t Device);
+} // namespace llvm::offload::tmp
+
int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
AsyncInfoTy &AsyncInfo) const {
// First, check if the user disabled atomic map transfer/malloc/dealloc.
@@ -65,10 +70,10 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
return OFFLOAD_SUCCESS;
}
-DeviceTy::DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID,
+DeviceTy::DeviceTy(int32_t DeviceID, ol_platform_handle_t Platform,
ol_device_handle_t DeviceHandle, PluginManager *PM)
- : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID),
- DeviceHandle(DeviceHandle), PM(PM), MappingInfo(*this) {}
+ : DeviceID(DeviceID), Platform(Platform), DeviceHandle(DeviceHandle),
+ PM(PM), MappingInfo(*this) {}
DeviceTy::~DeviceTy() {
if (DeviceID == -1 || !(getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE))
@@ -111,6 +116,13 @@ llvm::Error DeviceTy::initGeometryProps() {
if (auto Err = Query(DeviceInfo::WORK_GROUP_LOCAL_MEM_SIZE,
GeometryProps.MaxBlockSharedMemSize))
return Err;
+ // Not every device reports its number of compute units; zero turns off the
+ // heuristics that need it.
+ GeometryProps.NumComputeUnits =
+ getInfo<uint32_t>(DeviceInfo::NUM_COMPUTE_UNITS);
+ // Some defaults depend on the backend. If it is unknown, the generic ones
+ // apply.
+ GeometryProps.Backend = Backend;
return llvm::Error::success();
}
@@ -131,6 +143,14 @@ llvm::Error DeviceTy::init() {
IsHostDevice =
getInfo<ol_device_type_t>(DeviceInfo::TYPE) == OL_DEVICE_TYPE_HOST;
+ RTL = llvm::offload::tmp::__ol_tgt_GetPluginFromPlatform(Platform);
+ RTLDeviceID = llvm::offload::tmp::__ol_tgt_GetPluginDeviceId(DeviceHandle);
+
+ // Needs to be initialized before the GeometryProps.
+ if (olGetPlatformInfo(Platform, OL_PLATFORM_INFO_BACKEND, sizeof(Backend),
+ &Backend))
+ Backend = OL_PLATFORM_BACKEND_UNKNOWN;
+
getInfoString(DeviceInfo::UID, Uid);
if (Uid.empty())
ODBG(ODT_Init) << "Failed to get UID for device " << DeviceID;
>From 2b6e769dd3add8bbd8d8be60e59f8d7e5f30ec54 Mon Sep 17 00:00:00 2001
From: lijinpei-amd <jinpli at amd.com>
Date: Sat, 24 Oct 2026 17:50:49 +0800
Subject: [PATCH 18/37] [AMDGPU] Don't copy TargetSchedModel in
SIPreEmitPeephole (NFC) (#234860)
run() and collectUnpackingCandidates copied the subtarget's scheduling
model, including its ResourceFactors vector, once per block and once per
MFMA. Bind references instead.
Assisted-by: Claude Opus 5.5
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index b8fe4f8587ed90..d03acce575382b 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -715,7 +715,7 @@ void SIPreEmitPeephole::collectUnpackingCandidates(
auto *BB = BeginMI.getParent();
auto E = BB->end();
int TotalCyclesBetweenCandidates = 0;
- auto SchedModel = TII->getSchedModel();
+ const TargetSchedModel &SchedModel = TII->getSchedModel();
Register MFMADef = BeginMI.getOperand(0).getReg();
for (auto I = std::next(BeginMI.getIterator()); I != E; ++I) {
@@ -914,7 +914,7 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
for (MachineBasicBlock &MBB : MF) {
// Unpack packed instructions overlapped by MFMAs. This allows the
// compiler to co-issue unpacked instructions with MFMA
- auto SchedModel = TII->getSchedModel();
+ const TargetSchedModel &SchedModel = TII->getSchedModel();
SetVector<MachineInstr *> InstrsToUnpack;
for (auto &MI : make_early_inc_range(MBB.instrs())) {
if (!SIInstrInfo::isMFMA(MI))
>From ccda5d7abf91e00b91c743ed1e6a1cd200b42997 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Sat, 24 Oct 2026 13:09:31 +0200
Subject: [PATCH 19/37] [mlir][SPIR-V] Support MaxByteOffset decoration in
binary round trips (#235084)
---
mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp | 1 +
mlir/lib/Target/SPIRV/Serialization/Serializer.cpp | 1 +
mlir/test/Target/SPIRV/global-variable.mlir | 6 ++++--
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index 2297f0307be7d1..f658348ac6b6a9 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -314,6 +314,7 @@ LogicalResult spirv::Deserializer::processDecoration(ArrayRef<uint32_t> words) {
break;
}
case spirv::Decoration::Alignment:
+ case spirv::Decoration::MaxByteOffset:
case spirv::Decoration::DescriptorSet:
case spirv::Decoration::Binding:
case spirv::Decoration::Location:
diff --git a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
index e92ef88ed64371..94ad645a6ebebe 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
@@ -379,6 +379,7 @@ LogicalResult Serializer::processDecorationAttr(Location loc, uint32_t resultID,
{static_cast<uint32_t>(attr.getValue())});
});
case spirv::Decoration::Alignment:
+ case spirv::Decoration::MaxByteOffset:
case spirv::Decoration::Binding:
case spirv::Decoration::DescriptorSet:
case spirv::Decoration::Location:
diff --git a/mlir/test/Target/SPIRV/global-variable.mlir b/mlir/test/Target/SPIRV/global-variable.mlir
index 986e207c5ce24a..4982e2f55f35e7 100644
--- a/mlir/test/Target/SPIRV/global-variable.mlir
+++ b/mlir/test/Target/SPIRV/global-variable.mlir
@@ -19,12 +19,14 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
// -----
-// Alignment is also preserved on module-scope variables.
-spirv.module Physical64 OpenCL requires #spirv.vce<v1.0, [Addresses, Kernel, Linkage], []> {
+// Alignment and MaxByteOffset are also preserved on module-scope variables.
+spirv.module Physical64 OpenCL requires #spirv.vce<v1.1, [Addresses, Kernel, Linkage], []> {
// CHECK: spirv.GlobalVariable @aligned {alignment = 16 : i32} : !spirv.ptr<f32, CrossWorkgroup>
// CHECK: spirv.GlobalVariable @high_bit {alignment = -2147483648 : i32} : !spirv.ptr<f32, CrossWorkgroup>
+ // CHECK: spirv.GlobalVariable @bounded {max_byte_offset = 64 : i32} : !spirv.ptr<f32, CrossWorkgroup>
spirv.GlobalVariable @aligned {alignment = 16 : i32} : !spirv.ptr<f32, CrossWorkgroup>
spirv.GlobalVariable @high_bit {alignment = 2147483648 : i32} : !spirv.ptr<f32, CrossWorkgroup>
+ spirv.GlobalVariable @bounded {max_byte_offset = 64 : i32} : !spirv.ptr<f32, CrossWorkgroup>
}
// -----
>From 04b14262efc4b0c31c5571f5cd354e5b413d6f4c Mon Sep 17 00:00:00 2001
From: Morris Hafner <mmha at users.noreply.github.com>
Date: Sat, 24 Oct 2026 13:40:08 +0200
Subject: [PATCH 20/37] [CIR] Fix temporary cleanup boundaries for asm and
coroutines (#234664)
- Destroy asm operand temporaries at the end of the asm statement.
- Defer resume-expression temporary cleanups to the end of the enclosing
full-expression
- Clean up array-element default-argument temporaries before
constructing the next element, preserving lifetime extension.
- Keep conditional cleanup scopes open until the end of their owning
full-expression.
Note that this overlaps with #234303. I will rebase either depending on
which one gets merged first.
Assisted-By: Claude Opus 5.5
---
clang/include/clang/CIR/MissingFeatures.h | 1 -
clang/lib/CIR/CodeGen/CIRGenAsm.cpp | 2 +
clang/lib/CIR/CodeGen/CIRGenClass.cpp | 3 +-
clang/lib/CIR/CodeGen/CIRGenCleanup.cpp | 105 ++++-
clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp | 40 +-
clang/lib/CIR/CodeGen/CIRGenDecl.cpp | 2 +-
clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp | 12 +-
clang/lib/CIR/CodeGen/CIRGenFunction.h | 53 ++-
clang/test/CIR/CodeGen/array-ctor.cpp | 195 +++++++-
.../CIR/CodeGen/cleanup-conditional-eh.cpp | 335 ++++++++++++++
.../cleanup-conditional-with-wrapper.cpp | 61 ++-
.../test/CIR/CodeGen/cleanup-conditional.cpp | 281 ++++++++++++
clang/test/CIR/CodeGen/cleanup-scope-tmp.cpp | 96 +++-
clang/test/CIR/CodeGenCoroutines/coro-agg.cpp | 434 +++++++++++++++++-
14 files changed, 1562 insertions(+), 58 deletions(-)
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 7beb06541f7b64..ed7c2b9cfc92a1 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -119,7 +119,6 @@ struct MissingFeatures {
static bool noUniqueAddressLayout() { return false; }
// Coroutines
- static bool coroAwaitFullExprCleanups() { return false; }
static bool coroOutsideFrameMD() { return false; }
static bool coroGetRetObjManager() { return false; };
static bool coroUnwindRegion() { return false; };
diff --git a/clang/lib/CIR/CodeGen/CIRGenAsm.cpp b/clang/lib/CIR/CodeGen/CIRGenAsm.cpp
index 31b5ff07ecf641..c3c676bd742c12 100644
--- a/clang/lib/CIR/CodeGen/CIRGenAsm.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenAsm.cpp
@@ -261,6 +261,8 @@ static void emitAsmStores(CIRGenFunction &cgf, const AsmStmt &s,
}
mlir::LogicalResult CIRGenFunction::emitAsmStmt(const AsmStmt &s) {
+ FullExprCleanupScope cleanups(*this, &s);
+
// Assemble the final asm string.
std::string asmString = s.generateAsmString(getContext());
SourceLocation srcLoc = s.getAsmLoc();
diff --git a/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
index bcf301a0b5ae20..738f5464672435 100644
--- a/clang/lib/CIR/CodeGen/CIRGenClass.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
@@ -790,7 +790,8 @@ void CIRGenFunction::emitCXXAggrConstructorCall(
// destruction of every temporary created in a default argument expression
// is sequenced before the construction of the next array element, if any.
{
- RunCleanupsScope scope(*this);
+ FullExprCleanupScope scope(*this, e);
+ scope.deferLifetimeExtendedCleanups();
emitCXXConstructorCall(ctor, Ctor_Complete,
/*ForVirtualBase=*/false,
/*Delegating=*/false, currAVS, e);
diff --git a/clang/lib/CIR/CodeGen/CIRGenCleanup.cpp b/clang/lib/CIR/CodeGen/CIRGenCleanup.cpp
index 98fb9502ef0169..63e9dc24acbcfe 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCleanup.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCleanup.cpp
@@ -20,7 +20,7 @@
#include "CIRGenFunction.h"
#include "clang/AST/RecursiveASTVisitor.h"
-#include "clang/CIR/MissingFeatures.h"
+#include "llvm/Support/SaveAndRestore.h"
using namespace clang;
using namespace clang::CIRGen;
@@ -31,14 +31,39 @@ namespace {
class ConditionalEvaluationFinder
: public RecursiveASTVisitor<ConditionalEvaluationFinder> {
bool foundConditional = false;
+ bool inResumeExpr = false;
+
+ bool traverseSuspendExpr(CoroutineSuspendExpr *e) {
+ return TraverseStmt(e->getCommonExpr()) && traverseResumeExpr(e);
+ }
+
+ /// Traverse the resume expression of \p e, except for the binding of its
+ /// result, which emitSuspendExpression destroys after the cir.await.
+ bool traverseResumeExpr(CoroutineSuspendExpr *e) {
+ Expr *resume = e->getResumeExpr();
+ if (auto *bind = dyn_cast_if_present<CXXBindTemporaryExpr>(resume))
+ resume = bind->getSubExpr();
+ llvm::SaveAndRestore inResume(inResumeExpr, true);
+ return TraverseStmt(resume);
+ }
public:
+ static bool needsCleanupScope(const Stmt *s) {
+ ConditionalEvaluationFinder finder;
+ finder.TraverseStmt(const_cast<Stmt *>(s));
+ return finder.foundConditional;
+ }
+
+ static bool resumeExprDefersCleanups(const CoroutineSuspendExpr &s) {
+ ConditionalEvaluationFinder finder;
+ finder.traverseResumeExpr(const_cast<CoroutineSuspendExpr *>(&s));
+ return finder.foundConditional;
+ }
+
// Default arguments and default member initializers can contain conditional
// temporaries whose cleanups belong to the enclosing full-expression.
bool shouldVisitImplicitCode() const { return true; }
- bool found() const { return foundConditional; }
-
bool VisitAbstractConditionalOperator(AbstractConditionalOperator *) {
foundConditional = true;
return false;
@@ -74,19 +99,31 @@ class ConditionalEvaluationFinder
bool TraverseBlockExpr(BlockExpr *) { return true; }
bool TraverseStmtExpr(StmtExpr *) { return true; }
- // Skip over the implicit call to await_resume(). This requires cleanup
- // scopes for await full-expressions which don't exist yet.
- bool TraverseCoawaitExpr(CoawaitExpr *e) {
- assert(!cir::MissingFeatures::coroAwaitFullExprCleanups());
- return TraverseStmt(e->getOperand());
+ bool TraverseGCCAsmStmt(GCCAsmStmt *s) {
+ for (Stmt *operand : s->children())
+ if (!TraverseStmt(operand))
+ return false;
+ return true;
}
- bool TraverseDependentCoawaitExpr(DependentCoawaitExpr *e) {
- assert(!cir::MissingFeatures::coroAwaitFullExprCleanups());
- return TraverseStmt(e->getOperand());
+
+ // Traverse the common and resume expressions, whose temporaries belong to
+ // the enclosing full-expression. Ready and suspend have their own cleanup
+ // boundaries. The resume result's cleanup is handled separately.
+ bool TraverseCoawaitExpr(CoawaitExpr *e) { return traverseSuspendExpr(e); }
+ bool TraverseCoyieldExpr(CoyieldExpr *e) { return traverseSuspendExpr(e); }
+
+ bool VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *) {
+ if (!inResumeExpr)
+ return true;
+ foundConditional = true;
+ return false;
}
- bool TraverseCoyieldExpr(CoyieldExpr *e) {
- assert(!cir::MissingFeatures::coroAwaitFullExprCleanups());
- return TraverseStmt(e->getOperand());
+
+ bool VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *e) {
+ if (!inResumeExpr || !e->getType().isDestructedType())
+ return true;
+ foundConditional = true;
+ return false;
}
};
} // namespace
@@ -95,6 +132,10 @@ class ConditionalEvaluationFinder
// CIRGenFunction cleanup related
//===----------------------------------------------------------------------===//
+bool CIRGenFunction::resumeExprDefersCleanups(const CoroutineSuspendExpr &s) {
+ return ConditionalEvaluationFinder::resumeExprDefersCleanups(s);
+}
+
/// Emits all the code to cause the given temporary to be cleaned up.
void CIRGenFunction::emitCXXTemporary(const CXXTemporary *temporary,
QualType tempType, Address ptr) {
@@ -143,16 +184,16 @@ void CIRGenFunction::initFullExprCleanupWithFlag(Address activeFlag) {
}
CIRGenFunction::FullExprCleanupScope::FullExprCleanupScope(CIRGenFunction &cgf,
- const Expr *subExpr)
+ const Stmt *s)
: cgf(cgf), cleanups(cgf), scope(nullptr),
oldFullExprCleanupScope(cgf.currentFullExprCleanupScope),
+ oldOutermostConditional(cgf.outermostConditional),
+ oldHiddenOutermostConditional(cgf.hiddenOutermostConditional),
deferredCleanupStackSize(cgf.deferredConditionalCleanupStack.size()),
conditionalScopeDepth(cgf.conditionalCleanupScopes.size()) {
- assert(subExpr && "ExprWithCleanups always has a sub-expression");
- ConditionalEvaluationFinder finder;
- finder.TraverseStmt(const_cast<Expr *>(subExpr));
- if (finder.found()) {
+ assert(s && "expected an AST root for a cleanup boundary");
+ if (ConditionalEvaluationFinder::needsCleanupScope(s)) {
mlir::Location loc = cgf.builder.getUnknownLoc();
cir::CleanupKind cleanupKind = cgf.getLangOpts().Exceptions
? cir::CleanupKind::All
@@ -169,6 +210,13 @@ CIRGenFunction::FullExprCleanupScope::FullExprCleanupScope(CIRGenFunction &cgf,
// Conditional cleanup scopes are only opened while a full-expression scope
// is active to close them.
cgf.currentFullExprCleanupScope = scope;
+
+ // This full-expression owns its cleanups, even inside a conditional branch.
+ // Hide the outer conditional to keep those cleanups here, but remember it
+ // for lifetime-extended temporaries whose cleanups belong to an outer scope.
+ if (cgf.outermostConditional)
+ cgf.hiddenOutermostConditional = cgf.outermostConditional;
+ cgf.outermostConditional = nullptr;
}
/// If the alloca that backs \p addr is currently nested inside the body
@@ -303,6 +351,8 @@ void CIRGenFunction::FullExprCleanupScope::exit(
exited = true;
cgf.currentFullExprCleanupScope = oldFullExprCleanupScope;
+ cgf.outermostConditional = oldOutermostConditional;
+ cgf.hiddenOutermostConditional = oldHiddenOutermostConditional;
size_t oldSize = deferredCleanupStackSize;
@@ -599,7 +649,16 @@ static void setupCleanupBlockDeactivation(CIRGenFunction &cgf,
builder.createFlagStore(loc, false, var.getPointer());
}
-/// Deactive a cleanup that was created in an active state.
+static bool enclosesOpenConditionalCleanupScope(CIRGenFunction &cgf,
+ EHCleanupScope &scope) {
+ cir::CleanupScopeOp op = scope.getCleanupScopeOp();
+ return op &&
+ llvm::any_of(cgf.conditionalCleanupScopes, [&](const auto &condScope) {
+ return op->isProperAncestor(condScope.scope);
+ });
+}
+
+/// Deactivate a cleanup that was created in an active state.
void CIRGenFunction::deactivateCleanupBlock(EHScopeStack::stable_iterator c,
mlir::Operation *dominatingIP) {
assert(c != ehStack.stable_end() && "deactivating bottom of stack?");
@@ -607,9 +666,11 @@ void CIRGenFunction::deactivateCleanupBlock(EHScopeStack::stable_iterator c,
assert(scope.isActive() && "double deactivation");
// If it's the top of the stack, just pop it, but do so only if it belongs
- // to the current RunCleanupsScope.
+ // to the current RunCleanupsScope. If a conditional opened a cleanup scope
+ // keep it open till the end of the full-expression.
if (c == ehStack.stable_begin() &&
- currentCleanupStackDepth.strictlyEncloses(c)) {
+ currentCleanupStackDepth.strictlyEncloses(c) &&
+ !enclosesOpenConditionalCleanupScope(*this, scope)) {
popCleanupBlock(/*forDeactivation=*/true);
return;
}
diff --git a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
index 538db6747cf0ba..f0421572773b11 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
@@ -714,6 +714,22 @@ emitSuspendExpression(CIRGenFunction &cgf, CGCoroData &coro,
// throw.
const bool resumeInTry = isResumeExprInTryCatch(coro, kind, s);
+ // Resume-expression temporaries live until the enclosing full-expression
+ // ends. Defer their cleanups and guard them with construction flags, since
+ // destroying the coroutine while suspended skips their construction.
+ // The initial-suspend try/catch cleans up its temporaries in the try block.
+ std::optional<CIRGenFunction::ConditionalEvaluation> resumeEval;
+ const bool resumeDefersCleanups =
+ !resumeInTry && CIRGenFunction::resumeExprDefersCleanups(s);
+ if (resumeDefersCleanups && cgf.currentFullExprCleanupScope)
+ resumeEval.emplace(cgf, cgf.getLoc(s.getSourceRange()));
+ else if (resumeDefersCleanups)
+ cgf.cgm.errorNYI(s.getSourceRange(),
+ "resume expression cleanups without a full-expression "
+ "cleanup scope");
+ [[maybe_unused]] EHScopeStack::stable_iterator ehDepth =
+ cgf.ehStack.stable_begin();
+
// If the await_resume() result needs a destructor, take over its
// destruction, unless the destination owns it. The try/catch path destroys
// the result itself.
@@ -766,6 +782,8 @@ emitSuspendExpression(CIRGenFunction &cgf, CGCoroData &coro,
},
/*resumeBuilder=*/
[&](mlir::OpBuilder &b, mlir::Location loc) {
+ if (resumeEval)
+ resumeEval->beginEvaluation();
if (resumeInTry) {
// we are basically just emitting:
// resumeEh = false;
@@ -796,24 +814,27 @@ emitSuspendExpression(CIRGenFunction &cgf, CGCoroData &coro,
struct resumeEmitter final : CIRGenFunction::cxxTryBodyEmitter {
const CXXTryStmt &tryStmt;
+ const Expr *resumeExpr;
mlir::Location loc;
mlir::Value resumeEHVar;
- resumeEmitter(const CXXTryStmt &tryStmt, mlir::Location loc,
- Address resumeEHVar)
- : tryStmt(tryStmt), loc(loc),
+ resumeEmitter(const CXXTryStmt &tryStmt, const Expr *resumeExpr,
+ mlir::Location loc, Address resumeEHVar)
+ : tryStmt(tryStmt), resumeExpr(resumeExpr), loc(loc),
resumeEHVar(resumeEHVar.getPointer()) {}
mlir::LogicalResult operator()(CIRGenFunction &cgf) override {
- CIRGenFunction::RunCleanupsScope resumeCleanups(cgf);
+ // Destroy resume temporaries in the try block before marking
+ // resumption successful, so destructor exceptions are caught too.
+ CIRGenFunction::FullExprCleanupScope fullExpr(cgf, resumeExpr);
mlir::LogicalResult res =
cgf.emitStmt(tryStmt.getTryBlock(), /*useCurrentScope=*/true);
- resumeCleanups.forceCleanup();
+ fullExpr.exit();
cgf.getBuilder().createFlagStore(loc, true, resumeEHVar);
return res;
}
~resumeEmitter() override = default;
- } emitter{*tryStmt, mlirLoc, coro.resumeEHVar};
+ } emitter{*tryStmt, s.getResumeExpr(), mlirLoc, coro.resumeEHVar};
awaitBuild = cgf.emitCXXTryStmt(*tryStmt, emitter);
// We are not supposed to obtain the value from init suspend
@@ -852,10 +873,17 @@ emitSuspendExpression(CIRGenFunction &cgf, CGCoroData &coro,
}
}
+ if (resumeEval)
+ resumeEval->endEvaluation();
+
// Returns control back to parent.
cir::YieldOp::create(builder, loc);
});
+ assert((cgf.ehStack.stable_begin() == ehDepth ||
+ (resumeDefersCleanups && !resumeEval)) &&
+ "a resume expression cleanup was pushed in the resume region");
+
// Push the destructor right after the cir.await, where the result starts to
// exist, and not around it: destroying the coroutine at the suspend point
// would destroy the not yet constructed result. The cleanup runs at the end
diff --git a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
index 33c3a753cecd50..f74d7b643debb7 100644
--- a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
@@ -1332,7 +1332,7 @@ void CIRGenFunction::pushDestroyAndDeferDeactivation(
void CIRGenFunction::pushLifetimeExtendedDestroy(
CleanupKind cleanupKind, Address addr, QualType type, Destroyer *destroyer,
bool useEHCleanupForArray, const ValueDecl *extendingDecl) {
- if (isInConditionalBranch()) {
+ if (isInConditionalBranch() || hiddenOutermostConditional) {
cgm.errorNYI("conditional lifetime-extended destroy");
return;
}
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp b/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
index 7c5ce166edb059..ac186a5b58a38b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
@@ -869,10 +869,18 @@ void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
CIRGenFunction::ArrayInitLoopExprScope loopExprScope(
cgf, setArrayInitLoopExprScope, idx);
- if (arrayFiller)
+ // C++17 [class.temporary]p5:
+ // "when a default constructor is called to initialize an element of
+ // an array with no corresponding initializer [...] the destruction
+ // of every temporary created in a default argument is sequenced
+ // before the construction of the next array element, if any."
+ if (arrayFiller) {
+ CIRGenFunction::FullExprCleanupScope scope(cgf, arrayFiller);
+ scope.deferLifetimeExtendedCleanups();
emitInitializationToLValue(arrayFiller, elementLV);
- else
+ } else {
emitNullInitializationToLValue(loc, elementLV);
+ }
// Advance pointer and store them to temporary variable
cir::ConstantOp one = builder.getConstInt(
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 218b97ecb40356..396c10ec9001a5 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -1361,6 +1361,12 @@ class CIRGenFunction : public CIRGenTypeCache {
EHScopeStack::stable_iterator cleanupStackDepth, oldCleanupStackDepth;
size_t lifetimeExtendedCleanupStackSize;
CleanupDeactivationScope deactivateCleanups;
+ bool deferLifetimeExtended = false;
+
+ size_t oldLifetimeExtendedSize() const {
+ return deferLifetimeExtended ? cgf.lifetimeExtendedCleanupStack.size()
+ : lifetimeExtendedCleanupStackSize;
+ }
protected:
bool performCleanup;
@@ -1441,8 +1447,7 @@ class CIRGenFunction : public CIRGenTypeCache {
// pop cir.cleanup.scope ops); they already dominate the post-scope
// insertion point on their own.
if (didSpillAny) {
- cgf.popCleanupBlocks(cleanupStackDepth,
- lifetimeExtendedCleanupStackSize);
+ cgf.popCleanupBlocks(cleanupStackDepth, oldLifetimeExtendedSize());
// Reload the spilled values now that all cleanup popping (and
// promotion of any lifetime-extended cleanups onto the EH stack) is
@@ -1453,8 +1458,8 @@ class CIRGenFunction : public CIRGenTypeCache {
*valPtr = cgf.builder.createLoad(valPtr->getLoc(), addr);
}
} else {
- cgf.popCleanupBlocks(cleanupStackDepth,
- lifetimeExtendedCleanupStackSize, valuesToReload);
+ cgf.popCleanupBlocks(cleanupStackDepth, oldLifetimeExtendedSize(),
+ valuesToReload);
}
performCleanup = false;
@@ -1479,7 +1484,7 @@ class CIRGenFunction : public CIRGenTypeCache {
assert(performCleanup && "Already forced cleanup");
assert(deactivateCleanups.deactivated &&
"forceCleanupExceptLifetimeExtended() must be called first");
- cgf.popCleanupBlocks(cleanupStackDepth, lifetimeExtendedCleanupStackSize);
+ cgf.popCleanupBlocks(cleanupStackDepth, oldLifetimeExtendedSize());
performCleanup = false;
cgf.currentCleanupStackDepth = oldCleanupStackDepth;
}
@@ -1489,25 +1494,43 @@ class CIRGenFunction : public CIRGenTypeCache {
bool hasPendingCleanups() const {
return cgf.ehStack.stable_begin() != cleanupStackDepth;
}
+
+ /// Leave the cleanups of lifetime-extended temporaries created in this
+ /// scope to the enclosing scope instead of pushing them onto the EH stack
+ /// when this scope ends. This is for a scope that ends in the body of a
+ /// loop, such as the construction of an array element: the cleanup scope
+ /// of such a temporary ends with that of the variable that extends its
+ /// lifetime, after the loop.
+ void deferLifetimeExtendedCleanups() { deferLifetimeExtended = true; }
};
// Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
EHScopeStack::stable_iterator currentCleanupStackDepth = ehStack.stable_end();
+ class ConditionalEvaluation;
+
class FullExprCleanupScope {
CIRGenFunction &cgf;
RunCleanupsScope cleanups;
cir::CleanupScopeOp scope;
cir::CleanupScopeOp oldFullExprCleanupScope;
+ ConditionalEvaluation *oldOutermostConditional;
+ ConditionalEvaluation *oldHiddenOutermostConditional;
size_t deferredCleanupStackSize;
size_t conditionalScopeDepth;
bool exited = false;
public:
- FullExprCleanupScope(CIRGenFunction &cgf, const Expr *subExpr);
+ /// \p s is the AST root for one cleanup boundary: a full-expression, an
+ /// asm statement, or the construction of one array element.
+ FullExprCleanupScope(CIRGenFunction &cgf, const Stmt *s);
void exit(ArrayRef<mlir::Value *> valuesToReload = {});
+ void deferLifetimeExtendedCleanups() {
+ cleanups.deferLifetimeExtendedCleanups();
+ }
+
~FullExprCleanupScope() {
if (!exited)
exit();
@@ -1518,6 +1541,11 @@ class CIRGenFunction : public CIRGenTypeCache {
void operator=(const FullExprCleanupScope &) = delete;
};
+ /// Whether the resume expression of \p s contains a construct that defers
+ /// cleanups to the end of the enclosing full-expression: a temporary with a
+ /// destructor, other than the result of await_resume(), or a conditional.
+ static bool resumeExprDefersCleanups(const CoroutineSuspendExpr &s);
+
/// Captures cleanups for variables whose lifetime ends in a region owned by
/// an enclosing op rather than at the end of a cir.cleanup.scope, so that
/// they can be emitted into that region later. Examples for uses are:
@@ -2713,6 +2741,13 @@ class CIRGenFunction : public CIRGenTypeCache {
// we know if a temporary should be destroyed conditionally.
ConditionalEvaluation *outermostConditional = nullptr;
+ // The outermost conditional that a FullExprCleanupScope in a conditional
+ // branch hides from isInConditionalBranch(), if any. The cleanups of that
+ // full-expression run at its end, so they are not conditional, but a
+ // temporary whose lifetime is extended beyond it is still created
+ // conditionally.
+ ConditionalEvaluation *hiddenOutermostConditional = nullptr;
+
/// An RAII object to record that we're evaluating a statement
/// expression.
class StmtExprEvaluation {
@@ -2722,15 +2757,19 @@ class CIRGenFunction : public CIRGenTypeCache {
/// statement expression aren't conditional just because the
/// StmtExpr is.
ConditionalEvaluation *savedOutermostConditional;
+ ConditionalEvaluation *savedHiddenOutermostConditional;
public:
StmtExprEvaluation(CIRGenFunction &cgf)
- : cgf(cgf), savedOutermostConditional(cgf.outermostConditional) {
+ : cgf(cgf), savedOutermostConditional(cgf.outermostConditional),
+ savedHiddenOutermostConditional(cgf.hiddenOutermostConditional) {
cgf.outermostConditional = nullptr;
+ cgf.hiddenOutermostConditional = nullptr;
}
~StmtExprEvaluation() {
cgf.outermostConditional = savedOutermostConditional;
+ cgf.hiddenOutermostConditional = savedHiddenOutermostConditional;
}
};
diff --git a/clang/test/CIR/CodeGen/array-ctor.cpp b/clang/test/CIR/CodeGen/array-ctor.cpp
index 0c2e9652bdccb6..872b2dda8e2cf7 100644
--- a/clang/test/CIR/CodeGen/array-ctor.cpp
+++ b/clang/test/CIR/CodeGen/array-ctor.cpp
@@ -1,10 +1,12 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir -mmlir --mlir-print-ir-before=cir-lowering-prepare %s -o - 2>&1 | FileCheck --check-prefixes=CIR-BEFORE-LPP %s
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
-// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-linux-gnu -fclangir -fcxx-exceptions -fexceptions -emit-cir %s -o %t.eh.cir
+// RUN: FileCheck --input-file=%t.eh.cir %s --check-prefixes=TEMP,TEMP-EH
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir -mmlir --mlir-print-ir-before=cir-lowering-prepare %s -o - 2>&1 | FileCheck --check-prefixes=CIR-BEFORE-LPP %s
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR,TEMP,TEMP-NOEH
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM,TEMP-IR,TEMP-LLVM
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG,TEMP-IR
struct S {
S();
@@ -345,3 +347,182 @@ void Temp2InArray() {
// OGCG-NEXT: call void @_ZN5Temp2D1Ev({{.*}}[[TMP]])
// OGCG: %[[DTOR_ELT:.*]] = getelementptr inbounds %struct.CausesTemp2, ptr %{{.*}}, i64 -1
// OGCG-NEXT: call void @_ZN11CausesTemp2D1Ev(ptr {{.*}}%[[DTOR_ELT]])
+
+// Constructor default arguments are destroyed before the next array element.
+// Filler and capture-copy loops have the same boundary. Lifetime-extended
+// temporaries retain their enclosing cleanup ownership.
+
+struct ArgTemp {
+ ArgTemp(int);
+ ~ArgTemp();
+};
+
+extern "C" bool flag();
+extern "C" int g(const ArgTemp &);
+extern "C" void use(int);
+
+struct E {
+ E(const ArgTemp & = ArgTemp(1));
+ ~E();
+};
+
+struct F {
+ F(int = flag() ? g(ArgTemp(2)) : 0);
+ ~F();
+};
+
+extern "C" int take(F *);
+
+// TEMP-LABEL: cir.func {{.*}} @new_in_conditional(
+// TEMP: %[[TMP:.*]] = cir.alloca "ref.tmp0"
+// TEMP: %[[FLAG:.*]] = cir.alloca "cleanup.cond"
+// TEMP: cir.ternary(
+// TEMP: cir.do {
+// TEMP: cir.call @flag()
+// TEMP: %[[FALSE:.*]] = cir.const #false
+// TEMP-NEXT: cir.store %[[FALSE]], %[[FLAG]]
+// TEMP: cir.ternary(
+// TEMP: cir.call @_ZN7ArgTempC1Ei(%[[TMP]],
+// TEMP-NEXT: %[[TRUE:.*]] = cir.const #true
+// TEMP-NEXT: cir.store %[[TRUE]], %[[FLAG]]
+// TEMP: cir.call @_ZN1FC1Ei(
+// TEMP-NOEH: } cleanup normal {
+// TEMP-EH: } cleanup all {
+// TEMP: %[[LIVE:.*]] = cir.load {{.*}} %[[FLAG]]
+// TEMP-NEXT: cir.if %[[LIVE]] {
+// TEMP-NEXT: cir.call @_ZN7ArgTempD1Ev(%[[TMP]])
+// TEMP: } while {
+// TEMP: cir.call @take(
+// TEMP: cir.call @use(
+
+// TEMP-IR-LABEL: define {{.*}} void @new_in_conditional(
+// TEMP-IR: call void @_ZN7ArgTempC1Ei(ptr {{.*}} %[[TMP:[^,]+]], i32 {{.*}} 2)
+// TEMP-IR: call void @_ZN1FC1Ei(
+// TEMP-IR: call void @_ZN7ArgTempD1Ev(ptr {{.*}} %[[TMP]])
+// TEMP-IR: call i32 @take(
+// TEMP-IR: call void @use(
+
+extern "C" void new_in_conditional(bool b) { use(b ? take(new F[2]) : 0); }
+
+struct E3 {
+ E e[3];
+};
+
+extern "C" int take3(const E3 &);
+
+// TEMP-LABEL: cir.func {{.*}} @filler_in_conditional(
+// TEMP: %[[EXPLICIT:.*]] = cir.alloca "ref.tmp1"
+// TEMP: cir.ternary(
+// TEMP: cir.call @_ZN7ArgTempC1Ei(%[[EXPLICIT]],
+// TEMP: cir.do {
+// TEMP: cir.call @_ZN7ArgTempC1Ei(%[[TMP:[^,]+]],
+// TEMP: cir.cleanup.scope {
+// TEMP: cir.call @_ZN1EC1ERK7ArgTemp(
+// TEMP-NOEH: } cleanup normal {
+// TEMP-EH: } cleanup all {
+// TEMP-NEXT: cir.call @_ZN7ArgTempD1Ev(%[[TMP]])
+// TEMP: } while {
+// TEMP: cir.call @take3(
+// TEMP: cir.call @use(
+// TEMP: cir.call @_ZN2E3D1Ev(
+// TEMP: cir.call @_ZN7ArgTempD1Ev(%[[EXPLICIT]])
+
+// TEMP-IR-LABEL: define {{.*}} void @filler_in_conditional(
+// TEMP-IR: call void @_ZN7ArgTempC1Ei(ptr {{.*}} %[[EXPLICIT:[^,]+]], i32 {{.*}} 5)
+// TEMP-IR: call void @_ZN7ArgTempC1Ei(ptr {{.*}} %[[TMP:[^,]+]], i32 {{.*}} 1)
+// TEMP-IR: call void @_ZN1EC1ERK7ArgTemp(
+// TEMP-IR: call void @_ZN7ArgTempD1Ev(ptr {{.*}} %[[TMP]])
+// TEMP-IR: call i32 @take3(
+// TEMP-IR: call void @use(
+// TEMP-IR: call void @_ZN2E3D1Ev(
+// TEMP-IR: call void @_ZN7ArgTempD1Ev(ptr {{.*}} %[[EXPLICIT]])
+
+extern "C" void filler_in_conditional(bool b) { use(b ? take3(E3{{E(ArgTemp(5))}}) : 0); }
+
+struct C {
+ C();
+ C(const C &, const ArgTemp & = ArgTemp(3));
+ ~C();
+};
+
+// TEMP-LABEL: cir.func {{.*}} @capture(
+// TEMP: %[[TMP:.*]] = cir.alloca "ref.tmp0"
+// TEMP: cir.do {
+// TEMP: cir.call @_ZN7ArgTempC1Ei(%[[TMP]],
+// TEMP: cir.cleanup.scope {
+// TEMP: cir.call @_ZN1CC1ERKS_RK7ArgTemp(
+// TEMP-NOEH: } cleanup normal {
+// TEMP-EH: } cleanup all {
+// TEMP-NEXT: cir.call @_ZN7ArgTempD1Ev(%[[TMP]])
+// TEMP: } while {
+
+// TEMP-IR-LABEL: define {{.*}} void @capture(
+// TEMP-IR: call void @_ZN7ArgTempC1Ei(ptr {{.*}} %[[TMP:.*]], i32 {{.*}} 3)
+// TEMP-IR: call void @_ZN1CC1ERKS_RK7ArgTemp(
+// TEMP-IR: call void @_ZN7ArgTempD1Ev(ptr {{.*}} %[[TMP]])
+
+extern "C" void capture() {
+ C arr[2];
+ (void)[arr] {};
+}
+
+struct R {
+ const ArgTemp &s = ArgTemp(4);
+ int x;
+};
+
+extern "C" void use_r(R *);
+
+// TEMP-LABEL: cir.func {{.*}} @extended_member_filler(
+// TEMP: %[[TMP:.*]] = cir.alloca "ref.tmp0"
+// TEMP: cir.do {
+// TEMP: cir.call @_ZN7ArgTempC1Ei(%[[TMP]],
+// TEMP: } while {
+// TEMP: cir.call @use_r(
+// TEMP-NOEH: } cleanup normal {
+// TEMP-EH: } cleanup all {
+// TEMP-NEXT: cir.call @_ZN7ArgTempD1Ev(%[[TMP]])
+
+// TEMP-LLVM-LABEL: define {{.*}} void @extended_member_filler(
+// TEMP-LLVM: call void @_ZN7ArgTempC1Ei(ptr {{.*}} %[[TMP:.*]], i32 {{.*}} 4)
+// TEMP-LLVM-NOT: call void @_ZN7ArgTempD1Ev
+// TEMP-LLVM: call void @use_r(
+// TEMP-LLVM: call void @_ZN7ArgTempD1Ev(ptr {{.*}} %[[TMP]])
+
+extern "C" void extended_member_filler() {
+ R a[2] = {};
+ use_r(a);
+}
+
+struct Range {
+ int *begin();
+ int *end();
+};
+
+extern "C" Range take_range(E *);
+
+// TEMP-LABEL: cir.func {{.*}} @extended_default_arg(
+// TEMP: %[[TMP:.*]] = cir.alloca "ref.tmp1"
+// TEMP: cir.do {
+// TEMP: cir.call @_ZN7ArgTempC1Ei(%[[TMP]],
+// TEMP: cir.call @_ZN1EC1ERK7ArgTemp(
+// TEMP: } while {
+// TEMP: cir.call @take_range(
+// TEMP: cir.for : cond {
+// TEMP: cir.call @use(
+// TEMP: } step {
+// TEMP-NOEH: } cleanup normal {
+// TEMP-EH: } cleanup all {
+// TEMP-NEXT: cir.call @_ZN7ArgTempD1Ev(%[[TMP]])
+
+// TEMP-LLVM-LABEL: define {{.*}} void @extended_default_arg(
+// TEMP-LLVM: call void @_ZN7ArgTempC1Ei(ptr {{.*}} %[[TMP:.*]], i32 {{.*}} 1)
+// TEMP-LLVM: call void @take_range(
+// TEMP-LLVM-NOT: call void @_ZN7ArgTempD1Ev
+// TEMP-LLVM: call void @use(
+// TEMP-LLVM: call void @_ZN7ArgTempD1Ev(ptr {{.*}} %[[TMP]])
+
+extern "C" void extended_default_arg() {
+ for (int i : take_range(new E[2]))
+ use(i);
+}
diff --git a/clang/test/CIR/CodeGen/cleanup-conditional-eh.cpp b/clang/test/CIR/CodeGen/cleanup-conditional-eh.cpp
index 1423f087240d2f..f66dc5872d6d46 100644
--- a/clang/test/CIR/CodeGen/cleanup-conditional-eh.cpp
+++ b/clang/test/CIR/CodeGen/cleanup-conditional-eh.cpp
@@ -766,3 +766,338 @@ void test_flag_cleared_before_cond_cleanup() {
// OGCG: call void @_ZN7PayloadD1Ev(ptr {{.*}} %[[AGG_TMP]])
// OGCG: call void @_ZN5GuardD1Ev(ptr {{.*}} %[[REF_TMP]])
// OGCG: resume { ptr, i32 }
+
+// The members that an aggregate initialization has constructed so far are
+// destroyed when a later member initializer is left early, here by a return
+// in a statement expression in an operand of a conditional operator. The
+// member cleanups are deactivated at the end of the initialization, while the
+// cleanup scope that the conditional operator opened for its operands is still
+// open: the flag that guards the cleanup of the first member is cleared inside
+// that scope.
+
+struct Obj {
+ char c;
+ Obj(char);
+ ~Obj();
+};
+
+struct Pair {
+ Obj a;
+ Obj b;
+};
+
+Obj copy(const Obj &);
+Obj make(bool);
+
+void struct_ternary(bool b, bool c) {
+ Pair p{Obj{'a'}, b ? ({
+ if (c)
+ return;
+ Obj{'b'};
+ })
+ : Obj{'c'}};
+}
+
+// The cleanup scope of the conditional operator is empty and canonicalized
+// away.
+//
+// CIR-LABEL: cir.func {{.*}} @_Z14struct_ternarybb(
+// CIR: %[[P:.*]] = cir.alloca "p"
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive"
+// CIR: %[[A:.*]] = cir.get_member %[[P]][0] {name = "a"}
+// CIR: cir.call @_ZN3ObjC1Ec(%[[A]],
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]]
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.if
+// CIR: cir.if
+// CIR-NEXT: cir.return
+// CIR: } else {
+// CIR: cir.call @_ZN3ObjC1Ec(
+// CIR-NEXT: } loc(
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]]
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]]
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[A]])
+// CIR: cir.call @_ZN4PairD1Ev(%[[P]])
+
+// A logical operator opens such a scope too.
+
+void logical_and(bool b, bool c) {
+ Pair p{Obj{'a'}, make(b && ({
+ if (c)
+ return;
+ true;
+ }))};
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z11logical_andbb(
+// CIR: %[[P:.*]] = cir.alloca "p"
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive"
+// CIR: %[[A:.*]] = cir.get_member %[[P]][0] {name = "a"}
+// CIR: cir.store %{{.*}}, %[[ACTIVE]]
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.ternary
+// CIR: cir.return
+// CIR: cir.call @_Z4makeb(
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]]
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]]
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[A]])
+
+// The cleanup of an initialized member or init-capture, or the operator delete
+// or partial-array cleanup of a new-expression, is deactivated once the object
+// is initialized.
+// If an operand of a later ?:, && or || in the same initializer created a
+// temporary, the cleanup scope for that temporary is open in the body of the
+// deactivated cleanup's scope. It has to stay open until the end of the
+// full-expression, so the deactivated cleanup gets an active flag instead of
+// ending its scope early.
+
+int h(const Obj &);
+int g();
+void use(const Pair &, int);
+
+void aggregate(bool b) {
+ use(Pair{Obj{'a'}, b ? copy(Obj{'t'}) : Obj{'c'}}, g());
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z9aggregateb(
+// CIR: %[[PAIR:.*]] = cir.alloca "ref.tmp0" {{.*}} : !cir.ptr<!rec_Pair>
+// CIR: %[[TMP:.*]] = cir.alloca "ref.tmp1" {{.*}} : !cir.ptr<!rec_Obj>
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[A:.*]] = cir.get_member %[[PAIR]][0] {name = "a"}
+// CIR: cir.call @_ZN3ObjC1Ec(%[[A]], %{{.*}})
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.cleanup.scope {
+// CIR: cir.if %{{.*}} {
+// CIR: } else {
+// CIR-NEXT: %[[C:.*]] = cir.const #cir.int<99> : !s8i
+// CIR-NEXT: cir.call @_ZN3ObjC1Ec(%{{.*}}, %[[C]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR-NEXT: %[[G:.*]] = cir.call @_Z1gv()
+// CIR-NEXT: cir.call @_Z3useRK4Pairi(%[[PAIR]], %[[G]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: cir.call @_ZN4PairD1Ev(%[[PAIR]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_COND:.*]] = cir.load
+// CIR-NEXT: cir.if %[[IS_COND]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[TMP]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[A]])
+
+// LLVM-LABEL: define {{.*}} void @_Z9aggregateb(
+// LLVM: call void @_ZN3ObjC1Ec(ptr {{.*}} %[[A:.*]], i8 {{.*}} 97)
+// LLVM: invoke void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 116)
+// LLVM: invoke void @_Z4copyRK3Obj(
+// LLVM: invoke {{.*}} i32 @_Z1gv()
+// LLVM: invoke void @_Z3useRK4Pairi(
+// LLVM: call void @_ZN4PairD1Ev(
+// LLVM: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+
+// OGCG-LABEL: define {{.*}} void @_Z9aggregateb(
+// OGCG: call void @_ZN3ObjC1Ec(ptr {{.*}} %[[A:.*]], i8 {{.*}} 97)
+// OGCG: invoke void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 116)
+// OGCG: invoke void @_Z4copyRK3Obj(
+// OGCG: invoke {{.*}} i32 @_Z1gv()
+// OGCG: invoke void @_Z3useRK4Pairi(
+// OGCG: call void @_ZN4PairD1Ev(
+// OGCG: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+
+struct Agg {
+ Obj o;
+ int i;
+};
+
+void use_array(Agg *, int);
+
+void array_new(bool b) {
+ use_array(new Agg[2]{{Obj{'a'}, b ? h(Obj{'t'}) : 0}, {Obj{'c'}, 2}}, g());
+}
+
+// The member of the first element is deactivated with a flag, which keeps it
+// on the EH stack above the partial-array cleanup until the full-expression
+// ends.
+// CIR-LABEL: cir.func {{.*}} @_Z9array_newb(
+// CIR: %[[RESULT:.*]] = cir.alloca "__new_result"
+// CIR: %[[TMP:.*]] = cir.alloca "ref.tmp0" {{.*}} : !cir.ptr<!rec_Obj>
+// CIR: %[[COND:.*]] = cir.alloca "cleanup.cond" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[ARRAY:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!void> -> !cir.ptr<!rec_Agg>
+// CIR: %[[A:.*]] = cir.get_member %[[ARRAY]][0] {name = "o"}
+// CIR: cir.call @_ZN3ObjC1Ec(%[[A]], %{{.*}})
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.cleanup.scope {
+// CIR: cir.ternary
+// CIR: cir.call @_ZN3ObjC1Ec(%[[TMP]], %{{.*}})
+// CIR: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: %[[C:.*]] = cir.const #cir.int<99> : !s8i
+// CIR-NEXT: cir.call @_ZN3ObjC1Ec(%{{.*}}, %[[C]])
+// CIR: %[[NEW_ARRAY:.*]] = cir.load {{.*}} %[[RESULT]]
+// CIR-NEXT: %[[G:.*]] = cir.call @_Z1gv()
+// CIR-NEXT: cir.call @_Z9use_arrayP3Aggi(%[[NEW_ARRAY]], %[[G]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_COND:.*]] = cir.load {{.*}} %[[COND]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_COND]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[TMP]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[A]]){{.*}}
+
+// LLVM-LABEL: define {{.*}} void @_Z9array_newb(
+// LLVM: invoke void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 116)
+// LLVM: invoke void @_ZN3ObjC1Ec(ptr {{.*}}, i8 {{.*}} 99)
+// LLVM: invoke {{.*}} i32 @_Z1gv()
+// LLVM: invoke void @_Z9use_arrayP3Aggi(
+// LLVM: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+
+// OGCG-LABEL: define {{.*}} void @_Z9array_newb(
+// OGCG: invoke void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 116)
+// OGCG: invoke void @_ZN3ObjC1Ec(ptr {{.*}}, i8 {{.*}} 99)
+// OGCG: invoke {{.*}} i32 @_Z1gv()
+// OGCG: invoke void @_Z9use_arrayP3Aggi(
+// OGCG: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+
+struct Foo {
+ Foo(int);
+};
+
+void use_new(Foo *, int);
+
+void new_expr(bool b) { use_new(new Foo(b ? h(Obj{'x'}) : 0), g()); }
+
+// Without exceptions there is no operator delete cleanup to deactivate.
+// CIR-LABEL: cir.func {{.*}} @_Z8new_exprb(
+// CIR: %[[RESULT:.*]] = cir.alloca "__new_result"
+// CIR: %[[TMP:.*]] = cir.alloca "ref.tmp0" {{.*}} : !cir.ptr<!rec_Obj>
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[PTR:.*]] = cir.call @_Znwm(
+// CIR-NEXT: cir.cleanup.scope {
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: cir.cleanup.scope {
+// CIR: cir.ternary
+// CIR: cir.call @_ZN3FooC1Ei(
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: %[[FOO:.*]] = cir.load {{.*}} %[[RESULT]]
+// CIR-NEXT: %[[G:.*]] = cir.call @_Z1gv()
+// CIR-NEXT: cir.call @_Z7use_newP3Fooi(%[[FOO]], %[[G]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_COND:.*]] = cir.load
+// CIR-NEXT: cir.if %[[IS_COND]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[TMP]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup eh {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZdlPvm(%[[PTR]], %{{.*}})
+
+// LLVM-LABEL: define {{.*}} void @_Z8new_exprb(
+// LLVM: invoke void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 120)
+// LLVM: invoke void @_ZN3FooC1Ei(
+// LLVM: invoke {{.*}} i32 @_Z1gv()
+// LLVM: invoke void @_Z7use_newP3Fooi(
+// LLVM: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+
+// OGCG-LABEL: define {{.*}} void @_Z8new_exprb(
+// OGCG: invoke void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 120)
+// OGCG: invoke void @_ZN3FooC1Ei(
+// OGCG: invoke {{.*}} i32 @_Z1gv()
+// OGCG: invoke void @_Z7use_newP3Fooi(
+// OGCG: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+
+void use_int(int, int);
+
+void init_capture(bool b) {
+ use_int([o = Obj{'a'}, i = b ? h(Obj{'t'}) : 0] { return i; }(), g());
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z12init_captureb(
+// CIR: %[[LAMBDA:.*]] = cir.alloca "ref.tmp0"
+// CIR: %[[TMP:.*]] = cir.alloca "ref.tmp1" {{.*}} : !cir.ptr<!rec_Obj>
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[O:.*]] = cir.get_member %[[LAMBDA]][0] {name = "o"}
+// CIR: cir.call @_ZN3ObjC1Ec(%[[O]], %{{.*}})
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.cleanup.scope {
+// CIR: cir.ternary
+// CIR: cir.store {{.*}} : !s32i, !cir.ptr<!s32i>
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR-NEXT: %[[I:.*]] = cir.call @_ZZ12init_capturebENK3$_{{[0-9]+}}clEv(%[[LAMBDA]])
+// CIR-NEXT: %[[G:.*]] = cir.call @_Z1gv()
+// CIR-NEXT: cir.call @_Z7use_intii(%[[I]], %[[G]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: cir.call @_ZZ12init_capturebEN3$_{{[0-9]+}}D1Ev(%[[LAMBDA]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_COND:.*]] = cir.load
+// CIR-NEXT: cir.if %[[IS_COND]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[TMP]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup all {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[O]])
+
+// LLVM-LABEL: define {{.*}} void @_Z12init_captureb(
+// LLVM: invoke void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 116)
+// LLVM: invoke {{.*}} i32 @"_ZZ12init_capturebENK3$_{{[0-9]+}}clEv"(
+// LLVM: invoke {{.*}} i32 @_Z1gv()
+// LLVM: invoke void @_Z7use_intii(
+// LLVM: call void @"_ZZ12init_capturebEN3$_{{[0-9]+}}D1Ev"(
+// LLVM: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+// LLVM-LABEL: define internal {{.*}} @"_ZZ12init_capturebENK3$_{{[0-9]+}}clEv"(
+
+// OGCG-LABEL: define {{.*}} void @_Z12init_captureb(
+// OGCG: invoke void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 116)
+// OGCG: invoke {{.*}} i32 @"_ZZ12init_capturebENK3$_{{[0-9]+}}clEv"(
+// OGCG: invoke {{.*}} i32 @_Z1gv()
+// OGCG: invoke void @_Z7use_intii(
+// OGCG: call void @"_ZZ12init_capturebEN3$_{{[0-9]+}}D1Ev"(
+// OGCG: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+// OGCG-LABEL: define internal {{.*}} @"_ZZ12init_capturebENK3$_{{[0-9]+}}clEv"(
diff --git a/clang/test/CIR/CodeGen/cleanup-conditional-with-wrapper.cpp b/clang/test/CIR/CodeGen/cleanup-conditional-with-wrapper.cpp
index 1d3ac68bc5a8fa..0dd8c4e517f6a6 100644
--- a/clang/test/CIR/CodeGen/cleanup-conditional-with-wrapper.cpp
+++ b/clang/test/CIR/CodeGen/cleanup-conditional-with-wrapper.cpp
@@ -1,9 +1,15 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
-// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -fcxx-exceptions -fexceptions -emit-cir %s -o %t.eh.cir
+// RUN: FileCheck --input-file=%t.eh.cir %s --check-prefix=DEACT
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -fcxx-exceptions -fexceptions -emit-llvm %s -o %t.eh.ll
+// RUN: FileCheck --input-file=%t.eh.ll %s --check-prefix=DEACT-IR
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fcxx-exceptions -fexceptions -emit-llvm %s -o %t.ogcg.eh.ll
+// RUN: FileCheck --input-file=%t.ogcg.eh.ll %s --check-prefix=DEACT-IR
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR,DEACT
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM,DEACT-IR
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG,DEACT-IR
struct Base { ~Base(); };
@@ -349,3 +355,46 @@ void makeEntry() {
// OGCG: [[CLEANUP_DONE]]:
// OGCG: call void @_ZN4IterD1Ev(ptr {{.*}} %[[REF_TMP]])
// OGCG: ret void
+
+// A non-conditional array initializer leaves its temporary alive until the
+// enclosing full-expression ends.
+
+struct Obj {
+ char c;
+ Obj(char);
+ ~Obj();
+};
+
+extern "C" int h(const Obj &);
+extern "C" int g();
+
+struct Agg {
+ Obj o;
+ int i;
+};
+
+extern "C" void use_array(Agg *, int);
+
+// DEACT-LABEL: cir.func {{.*}} @array_new_temporary(
+// DEACT: %[[TMP:.*]] = cir.alloca "ref.tmp0"
+// DEACT: cir.call @_ZN3ObjC1Ec(
+// DEACT-NEXT: %[[TRUE:.*]] = cir.const #true
+// DEACT-NEXT: cir.store %[[TRUE]], %[[ACTIVE:.*]]
+// DEACT: cir.call @h(%[[TMP]])
+// DEACT: %[[FALSE:.*]] = cir.const #false
+// DEACT-NEXT: cir.store %[[FALSE]], %[[ACTIVE]]
+// DEACT-NOT: cir.call @_ZN3ObjD1Ev(%[[TMP]])
+// DEACT: cir.call @g()
+// DEACT-NEXT: cir.call @use_array(
+// DEACT: cir.call @_ZN3ObjD1Ev(%[[TMP]])
+
+// DEACT-IR-LABEL: define {{.*}} void @array_new_temporary(
+// DEACT-IR: {{(call|invoke)}} void @_ZN3ObjC1Ec(ptr {{.*}} %[[TMP:.*]], i8 {{.*}} 116)
+// DEACT-IR-NOT: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+// DEACT-IR: {{(call|invoke)}} i32 @g()
+// DEACT-IR: {{(call|invoke)}} void @use_array(
+// DEACT-IR: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[TMP]])
+
+extern "C" void array_new_temporary() {
+ use_array(new Agg[2]{{Obj{'a'}, h(Obj{'t'})}, {Obj{'c'}, 2}}, g());
+}
diff --git a/clang/test/CIR/CodeGen/cleanup-conditional.cpp b/clang/test/CIR/CodeGen/cleanup-conditional.cpp
index 1f86f1d1cd4831..b9df861d17e4f3 100644
--- a/clang/test/CIR/CodeGen/cleanup-conditional.cpp
+++ b/clang/test/CIR/CodeGen/cleanup-conditional.cpp
@@ -1356,3 +1356,284 @@ void lambda_init_capture() {
// OGCG: [[DTOR]]:
// OGCG-NEXT: call void @_ZN1SD1Ev(ptr {{.*}} %[[TMP]])
// OGCG: call void @"_ZZ19lambda_init_capturevENK3$_0clEv"(
+
+// The members that an aggregate initialization has constructed so far are
+// destroyed when a later member initializer is left early, here by a return
+// in a statement expression in an operand of a conditional operator. The
+// member cleanups are deactivated at the end of the initialization, while the
+// cleanup scope that the conditional operator opened for its operands is still
+// open: the flag that guards the cleanup of the first member is cleared inside
+// that scope.
+
+struct Obj {
+ char c;
+ Obj(char);
+ ~Obj();
+};
+
+struct Pair {
+ Obj a;
+ Obj b;
+};
+
+Obj copy(const Obj &);
+Obj make(bool);
+
+void struct_ternary(bool b, bool c) {
+ Pair p{Obj{'a'}, b ? ({
+ if (c)
+ return;
+ Obj{'b'};
+ })
+ : Obj{'c'}};
+}
+
+// The cleanup scope of the conditional operator is empty and canonicalized
+// away.
+//
+// CIR-LABEL: cir.func {{.*}} @_Z14struct_ternarybb(
+// CIR: %[[P:.*]] = cir.alloca "p"
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive"
+// CIR: %[[A:.*]] = cir.get_member %[[P]][0] {name = "a"}
+// CIR: cir.call @_ZN3ObjC1Ec(%[[A]],
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]]
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.if
+// CIR: cir.if
+// CIR-NEXT: cir.return
+// CIR: } else {
+// CIR: cir.call @_ZN3ObjC1Ec(
+// CIR-NEXT: } loc(
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]]
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]]
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[A]])
+// CIR: cir.call @_ZN4PairD1Ev(%[[P]])
+
+// The return leaves through the cleanup, which destroys the first member.
+// Completing the initialization clears the flag.
+//
+// LLVMCIR-LABEL: define dso_local void @_Z14struct_ternarybb(
+// LLVMCIR: %[[A:.*]] = getelementptr inbounds nuw %struct.Pair, ptr %{{.*}}, i32 0, i32 0
+// LLVMCIR-NEXT: call void @_ZN3ObjC1Ec(ptr {{.*}} %[[A]], i8 noundef signext 97)
+// LLVMCIR-NEXT: store i8 1, ptr %[[ACTIVE:[0-9]+]]
+// LLVMCIR: store i32 1, ptr %[[SLOT:[0-9]+]]
+// LLVMCIR-NEXT: br label %[[CLEANUP:[0-9]+]]
+// LLVMCIR: call void @_ZN3ObjC1Ec(ptr {{.*}}, i8 noundef signext 98)
+// LLVMCIR: call void @_ZN3ObjC1Ec(ptr {{.*}}, i8 noundef signext 99)
+// LLVMCIR-NEXT: br label %[[END:[0-9]+]]
+// LLVMCIR: [[END]]:
+// LLVMCIR-NEXT: store i8 0, ptr %[[ACTIVE]]
+// LLVMCIR-NEXT: store i32 0, ptr %[[SLOT]]
+// LLVMCIR-NEXT: br label %[[CLEANUP]]
+// LLVMCIR: [[CLEANUP]]:
+// LLVMCIR-NEXT: %[[FLAG:.*]] = load i8, ptr %[[ACTIVE]]
+// LLVMCIR-NEXT: %[[IS_ACTIVE:.*]] = trunc i8 %[[FLAG]] to i1
+// LLVMCIR-NEXT: br i1 %[[IS_ACTIVE]], label %[[DESTROY:[0-9]+]], label %{{.*}}
+// LLVMCIR: [[DESTROY]]:
+// LLVMCIR-NEXT: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[A]])
+// LLVMCIR: call void @_ZN4PairD1Ev(
+// LLVMCIR: ret void
+
+// OGCG-LABEL: define dso_local void @_Z14struct_ternarybb(
+// OGCG: %[[A:.*]] = getelementptr inbounds nuw %struct.Pair, ptr %p, i32 0, i32 0
+// OGCG: if.then:
+// OGCG-NEXT: call void @_ZN3ObjD1Ev(ptr {{.*}} %[[A]])
+// OGCG-NEXT: br label %return
+// OGCG: cond.end:
+// OGCG-NEXT: call void @_ZN4PairD1Ev(
+
+// A logical operator opens such a scope too.
+
+void logical_and(bool b, bool c) {
+ Pair p{Obj{'a'}, make(b && ({
+ if (c)
+ return;
+ true;
+ }))};
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z11logical_andbb(
+// CIR: %[[P:.*]] = cir.alloca "p"
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive"
+// CIR: %[[A:.*]] = cir.get_member %[[P]][0] {name = "a"}
+// CIR: cir.store %{{.*}}, %[[ACTIVE]]
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.ternary
+// CIR: cir.return
+// CIR: cir.call @_Z4makeb(
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]]
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]]
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[A]])
+
+// LLVMCIR-LABEL: define dso_local void @_Z11logical_andbb(
+// LLVMCIR: call void @_Z4makeb(
+// LLVMCIR-NEXT: store i8 0, ptr %[[ACTIVE:[0-9]+]]
+// LLVMCIR: load i8, ptr %[[ACTIVE]]
+// LLVMCIR: call void @_ZN3ObjD1Ev(
+
+// OGCG-LABEL: define dso_local void @_Z11logical_andbb(
+// OGCG: if.then:
+// OGCG-NEXT: call void @_ZN3ObjD1Ev(
+
+// The cleanup of an initialized member or init-capture, or the operator delete
+// or partial-array cleanup of a new-expression, is deactivated once the object
+// is initialized.
+// If an operand of a later ?:, && or || in the same initializer created a
+// temporary, the cleanup scope for that temporary is open in the body of the
+// deactivated cleanup's scope. It has to stay open until the end of the
+// full-expression, so the deactivated cleanup gets an active flag instead of
+// ending its scope early.
+
+int h(const Obj &);
+int g();
+void use(const Pair &, int);
+
+void aggregate(bool b) {
+ use(Pair{Obj{'a'}, b ? copy(Obj{'t'}) : Obj{'c'}}, g());
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z9aggregateb(
+// CIR: %[[PAIR:.*]] = cir.alloca "ref.tmp0" {{.*}} : !cir.ptr<!rec_Pair>
+// CIR: %[[TMP:.*]] = cir.alloca "ref.tmp1" {{.*}} : !cir.ptr<!rec_Obj>
+// CIR: %[[COND:.*]] = cir.alloca "cleanup.cond" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[A:.*]] = cir.get_member %[[PAIR]][0] {name = "a"}
+// CIR: cir.call @_ZN3ObjC1Ec(%[[A]], %{{.*}})
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.cleanup.scope {
+// CIR: cir.if %{{.*}} {
+// CIR: cir.call @_ZN3ObjC1Ec(%[[TMP]], %{{.*}})
+// CIR: cir.call @_Z4copyRK3Obj(%{{.*}}, %[[TMP]])
+// CIR: } else {
+// CIR-NEXT: %[[C:.*]] = cir.const #cir.int<99> : !s8i
+// CIR-NEXT: cir.call @_ZN3ObjC1Ec(%{{.*}}, %[[C]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR-NEXT: %[[G:.*]] = cir.call @_Z1gv()
+// CIR-NEXT: cir.call @_Z3useRK4Pairi(%[[PAIR]], %[[G]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: cir.call @_ZN4PairD1Ev(%[[PAIR]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: %[[IS_COND:.*]] = cir.load {{.*}} %[[COND]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_COND]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[TMP]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[A]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.return
+
+struct Agg {
+ Obj o;
+ int i;
+};
+
+void use_array(Agg *, int);
+
+void array_new(bool b) {
+ use_array(new Agg[2]{{Obj{'a'}, b ? h(Obj{'t'}) : 0}, {Obj{'c'}, 2}}, g());
+}
+
+// The member of the first element is deactivated with a flag, which keeps it
+// on the EH stack above the partial-array cleanup. Without exceptions, that
+// cleanup is EH-only and therefore neither a normal nor an EH cleanup, so its
+// deactivation needs no flag.
+// CIR-LABEL: cir.func {{.*}} @_Z9array_newb(
+// CIR: %[[TMP:.*]] = cir.alloca "ref.tmp0" {{.*}} : !cir.ptr<!rec_Obj>
+// CIR: %[[COND:.*]] = cir.alloca "cleanup.cond" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[ARRAY:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!void> -> !cir.ptr<!rec_Agg>
+// CIR: %[[A:.*]] = cir.get_member %[[ARRAY]][0] {name = "o"}
+// CIR: cir.call @_ZN3ObjC1Ec(%[[A]], %{{.*}})
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.cleanup.scope {
+// CIR: cir.ternary
+// CIR: cir.call @_ZN3ObjC1Ec(%[[TMP]], %{{.*}})
+// CIR: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: %[[C:.*]] = cir.const #cir.int<99> : !s8i
+// CIR-NEXT: cir.call @_ZN3ObjC1Ec(%{{.*}}, %[[C]])
+// CIR: %[[G:.*]] = cir.call @_Z1gv()
+// CIR-NEXT: cir.call @_Z9use_arrayP3Aggi(%[[ARRAY]], %[[G]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: %[[IS_COND:.*]] = cir.load {{.*}} %[[COND]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_COND]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[TMP]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[A]]){{.*}}
+
+void use_int(int, int);
+
+void init_capture(bool b) {
+ use_int([o = Obj{'a'}, i = b ? h(Obj{'t'}) : 0] { return i; }(), g());
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z12init_captureb(
+// CIR: %[[LAMBDA:.*]] = cir.alloca "ref.tmp0"
+// CIR: %[[TMP:.*]] = cir.alloca "ref.tmp1" {{.*}} : !cir.ptr<!rec_Obj>
+// CIR: %[[ACTIVE:.*]] = cir.alloca "cleanup.isactive" {{.*}} : !cir.ptr<!cir.bool>
+// CIR: %[[O:.*]] = cir.get_member %[[LAMBDA]][0] {name = "o"}
+// CIR: cir.call @_ZN3ObjC1Ec(%[[O]], %{{.*}})
+// CIR-NEXT: %[[TRUE:.*]] = cir.const #true
+// CIR-NEXT: cir.store %[[TRUE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR: cir.cleanup.scope {
+// CIR: cir.ternary
+// CIR: cir.store {{.*}} : !s32i, !cir.ptr<!s32i>
+// CIR-NEXT: %[[FALSE:.*]] = cir.const #false
+// CIR-NEXT: cir.store %[[FALSE]], %[[ACTIVE]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR-NEXT: cir.cleanup.scope {
+// CIR-NEXT: %[[I:.*]] = cir.call @_ZZ12init_capturebENK3$_{{[0-9]+}}clEv(%[[LAMBDA]])
+// CIR-NEXT: %[[G:.*]] = cir.call @_Z1gv()
+// CIR-NEXT: cir.call @_Z7use_intii(%[[I]], %[[G]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: cir.call @_ZZ12init_capturebEN3$_{{[0-9]+}}D1Ev(%[[LAMBDA]])
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: %[[IS_COND:.*]] = cir.load
+// CIR-NEXT: cir.if %[[IS_COND]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[TMP]]){{.*}}
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: }
+// CIR-NEXT: cir.yield
+// CIR-NEXT: } cleanup normal {
+// CIR-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR-NEXT: cir.if %[[IS_ACTIVE]] {
+// CIR-NEXT: cir.call @_ZN3ObjD1Ev(%[[O]])
diff --git a/clang/test/CIR/CodeGen/cleanup-scope-tmp.cpp b/clang/test/CIR/CodeGen/cleanup-scope-tmp.cpp
index 5f5c3f67c136ca..e89ad0e10b41d2 100644
--- a/clang/test/CIR/CodeGen/cleanup-scope-tmp.cpp
+++ b/clang/test/CIR/CodeGen/cleanup-scope-tmp.cpp
@@ -1,5 +1,11 @@
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR,ASM
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -fcxx-exceptions -fexceptions -emit-cir %s -o %t.eh.cir
+// RUN: FileCheck --input-file=%t.eh.cir %s --check-prefix=ASM
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=ASM-IR
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=ASM-IR
struct StructWithDestructor {
~StructWithDestructor();
@@ -29,3 +35,91 @@ void cleanup_scope_with_body_and_cleanup() {
// CIR: cir.call @_ZN20StructWithDestructorD1Ev(%0) nothrow : (!cir.ptr<!rec_StructWithDestructor> {{.*}}) -> ()
// CIR: cir.yield
// CIR: }
+
+// The operands of an asm statement are not wrapped in an ExprWithCleanups.
+// Their temporaries are destroyed at the end of the asm statement.
+
+struct S {
+ S(int);
+ ~S();
+};
+
+extern "C" {
+
+int g(const S &);
+void use(int);
+
+void asm_plain() {
+ asm volatile("" : : "r"(g(S(1))));
+ use(0);
+}
+
+// ASM-LABEL: cir.func {{.*}} @asm_plain(
+// ASM: %[[TMP:.*]] = cir.alloca "ref.tmp0" {{.*}} : !cir.ptr<!rec_S>
+// ASM: cir.call @_ZN1SC1Ei(%[[TMP]],
+// ASM-NEXT: cir.cleanup.scope {
+// ASM-NEXT: %[[G:.*]] = cir.call @g(%[[TMP]])
+// ASM-NEXT: cir.asm(
+// ASM: in = [%[[G]] : !s32i],
+// ASM: cir.yield
+// ASM-NEXT: } cleanup {{(normal|all)}} {
+// ASM-NEXT: cir.call @_ZN1SD1Ev(%[[TMP]])
+// ASM-NEXT: cir.yield
+// ASM-NEXT: }
+// ASM-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// ASM-NEXT: cir.call @use(%[[ZERO]])
+
+// ASM-IR-LABEL: define {{.*}} void @asm_plain(
+// ASM-IR: call void @_ZN1SC1Ei(ptr {{.*}} %[[TMP:.*]], i32 {{.*}} 1)
+// ASM-IR: call void asm sideeffect
+// ASM-IR: call void @_ZN1SD1Ev(ptr {{.*}} %[[TMP]])
+// ASM-IR: call void @use(i32 {{.*}} 0)
+
+// The cleanup scope of the enclosing full-expression is open, but the asm
+// statement in the statement expression needs its own.
+void asm_in_statement_expression(bool b) {
+ use(g(S(3)) + (b ? 1 : 0) + ({
+ int y = 0;
+ asm volatile("" : "+r"(y) : "r"(b ? g(S(4)) : 0));
+ y;
+ }));
+}
+
+// ASM-LABEL: cir.func {{.*}} @asm_in_statement_expression(
+// ASM: %[[TMP3:.*]] = cir.alloca "ref.tmp0" {{.*}} : !cir.ptr<!rec_S>
+// ASM: %[[TMP4:.*]] = cir.alloca "ref.tmp1" {{.*}} : !cir.ptr<!rec_S>
+// ASM: %[[ACTIVE:.*]] = cir.alloca "cleanup.cond" {{.*}} : !cir.ptr<!cir.bool>
+// ASM: cir.scope {
+// ASM: cir.cleanup.scope {
+// ASM-NEXT: %[[FALSE:.*]] = cir.const #false
+// ASM-NEXT: cir.store %[[FALSE]], %[[ACTIVE]]
+// ASM-NEXT: %[[ARG:.*]] = cir.ternary(%{{.*}}, true {
+// ASM: cir.call @_ZN1SC1Ei(%[[TMP4]],
+// ASM-NEXT: %[[TRUE:.*]] = cir.const #true
+// ASM-NEXT: cir.store %[[TRUE]], %[[ACTIVE]]
+// ASM: cir.call @g(%[[TMP4]])
+// ASM: }) : (!cir.bool) -> !s32i
+// ASM-NEXT: cir.asm(
+// ASM: in = [%[[ARG]] : !s32i],
+// ASM: cir.yield
+// ASM-NEXT: } cleanup {{(normal|all)}} {
+// ASM-NEXT: %[[IS_ACTIVE:.*]] = cir.load {{.*}} %[[ACTIVE]]
+// ASM-NEXT: cir.if %[[IS_ACTIVE]] {
+// ASM-NEXT: cir.call @_ZN1SD1Ev(%[[TMP4]]){{.*}}
+// ASM-NEXT: }
+// ASM-NEXT: cir.yield
+// ASM-NEXT: }
+// ASM: }
+// ASM: cir.call @use(
+// ASM: } cleanup {{(normal|all)}} {
+// ASM-NEXT: cir.call @_ZN1SD1Ev(%[[TMP3]])
+
+// ASM-IR-LABEL: define {{.*}} void @asm_in_statement_expression(
+// ASM-IR: call void @_ZN1SC1Ei(ptr {{.*}} %[[TMP3:.*]], i32 {{.*}} 3)
+// ASM-IR: call void @_ZN1SC1Ei(ptr {{.*}} %[[TMP4:.*]], i32 {{.*}} 4)
+// ASM-IR: call i32 asm sideeffect
+// ASM-IR: call void @_ZN1SD1Ev(ptr {{.*}} %[[TMP4]])
+// ASM-IR: call void @use(
+// ASM-IR: call void @_ZN1SD1Ev(ptr {{.*}} %[[TMP3]])
+
+} // extern "C"
diff --git a/clang/test/CIR/CodeGenCoroutines/coro-agg.cpp b/clang/test/CIR/CodeGenCoroutines/coro-agg.cpp
index 47791d72627716..3c09be504168f5 100644
--- a/clang/test/CIR/CodeGenCoroutines/coro-agg.cpp
+++ b/clang/test/CIR/CodeGenCoroutines/coro-agg.cpp
@@ -1,11 +1,11 @@
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -Wno-coroutine-missing-unhandled-exception -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s -check-prefixes=CIR,CIR-COMMON,CIR-NOEH
+// RUN: FileCheck --input-file=%t.cir %s -check-prefixes=CIR,CIR-COMMON,CIR-NOEH,RESUME,RESUME-NOEH
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -fcxx-exceptions -fexceptions -Wno-coroutine-missing-unhandled-exception -emit-cir %s -o %t.eh.cir
-// RUN: FileCheck --input-file=%t.eh.cir %s -check-prefixes=CIR-EH,CIR-COMMON
+// RUN: FileCheck --input-file=%t.eh.cir %s -check-prefixes=CIR-EH,CIR-COMMON,RESUME,RESUME-EH
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -emit-llvm -disable-llvm-passes -Wno-coroutine-missing-unhandled-exception %s -o %t.ll
-// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+// RUN: FileCheck --input-file=%t.ll %s -check-prefixes=OGCG,RESUME-IR
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fcxx-exceptions -fexceptions -emit-llvm -disable-llvm-passes -Wno-coroutine-missing-unhandled-exception %s -o %t.eh.ll
-// RUN: FileCheck --input-file=%t.eh.ll %s -check-prefix=OGCG-EH
+// RUN: FileCheck --input-file=%t.eh.ll %s -check-prefixes=OGCG-EH,RESUME-IR
#include "Inputs/coroutine.h"
@@ -589,3 +589,429 @@ ThrowingDtorInitTask throwing_dtor_initial_resume() {
// OGCG-EH-NEXT: to label %[[CONT:.*]] unwind label
// OGCG-EH: [[CONT]]:
// OGCG-EH-NEXT: store i1 false, ptr %[[RESUME_EH]], align 1
+
+// Resume-expression temporaries survive through the enclosing full-expression.
+// Construction guards exclude paths that destroy the coroutine while suspended;
+// array elements and ready/suspend calls retain their own cleanup boundaries.
+
+struct S {
+ S(int);
+ ~S();
+};
+
+struct Agg {
+ S s;
+ int i;
+};
+
+extern "C" bool resume_flag();
+extern "C" int resume_arg(const S &);
+extern "C" int resume_next();
+extern "C" void resume_use(int);
+extern "C" void resume_use_pair(int, int);
+extern "C" void resume_use_result(NT);
+extern "C" void resume_use_aggregate(const Agg &, int);
+extern "C" void mark();
+
+struct DA {
+ bool await_ready();
+ void await_suspend(std::coroutine_handle<>);
+ int await_resume(const S & = S(1));
+};
+
+struct DV {
+ bool await_ready();
+ void await_suspend(std::coroutine_handle<>);
+ int await_resume(S = S(2));
+};
+
+struct DCC {
+ bool await_ready();
+ void await_suspend(std::coroutine_handle<>);
+ int await_resume(int = resume_flag() ? resume_arg(S(1)) : 0);
+};
+
+struct RA {
+ bool await_ready();
+ void await_suspend(std::coroutine_handle<>);
+ NT await_resume(const S & = S(1));
+};
+
+struct PL {
+ bool await_ready();
+ void await_suspend(std::coroutine_handle<>);
+ int await_resume();
+};
+
+struct Task {
+ struct promise_type {
+ Task get_return_object();
+ std::suspend_never initial_suspend();
+ std::suspend_never final_suspend() noexcept;
+ void return_void();
+ void unhandled_exception();
+ DA yield_value(int);
+ };
+};
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @defarg_condtemp(
+// RESUME: %[[TMP:[^,]+]] = cir.alloca "ref.tmp{{[0-9]+}}" {{.*}} : !cir.ptr<!rec_S>
+// RESUME: %[[FLAG:.*]] = cir.alloca "cleanup.cond"
+// RESUME: %[[FALSE:.*]] = cir.const #false
+// RESUME-NEXT: cir.store %[[FALSE]], %[[FLAG]]
+// RESUME: cir.await(user, ready : {
+// RESUME: }, resume : {
+// RESUME: cir.ternary(
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP]],
+// RESUME-NEXT: %[[TRUE:.*]] = cir.const #true
+// RESUME-NEXT: cir.store %[[TRUE]], %[[FLAG]]
+// RESUME: cir.call @_ZN3DCC12await_resumeEi(
+// RESUME: },)
+// RESUME: cir.call @resume_use(
+// RESUME-NOEH: } cleanup normal {
+// RESUME-EH: } cleanup all {
+// RESUME: %[[LIVE:.*]] = cir.load {{.*}} %[[FLAG]]
+// RESUME-NEXT: cir.if %[[LIVE]] {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[TMP]])
+
+extern "C" Task defarg_condtemp() {
+ resume_use(co_await DCC{});
+}
+
+// Cover reference and value arguments across two suspension points. Destroy
+// them in reverse order at full-expression end, before the next statement.
+// Construction flags exclude paths that destroy the coroutine while suspended.
+// RESUME-LABEL: cir.func coroutine {{.*}} @two_awaits(
+// RESUME-DAG: %[[TMP1:[^,]+]] = cir.alloca "ref.tmp{{[0-9]+}}" {{.*}} : !cir.ptr<!rec_S>
+// RESUME-DAG: %[[FLAG1:.*]] = cir.alloca "cleanup.cond"
+// RESUME-DAG: %[[TMP2:[^,]+]] = cir.alloca "agg.tmp{{[0-9]+}}" {{.*}} : !cir.ptr<!rec_S>
+// RESUME-DAG: %[[FLAG2:.*]] = cir.alloca "cleanup.cond"
+// RESUME: %[[FALSE1:.*]] = cir.const #false
+// RESUME-NEXT: cir.store %[[FALSE1]], %[[FLAG1]]
+// RESUME: cir.await(user, ready : {
+// RESUME: }, resume : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP1]],
+// RESUME-NEXT: %[[TRUE1:.*]] = cir.const #true
+// RESUME-NEXT: cir.store %[[TRUE1]], %[[FLAG1]]
+// RESUME: cir.call @_ZN2DA12await_resumeERK1S(
+// RESUME: },)
+// RESUME: %[[FALSE2:.*]] = cir.const #false
+// RESUME-NEXT: cir.store %[[FALSE2]], %[[FLAG2]]
+// RESUME: cir.await(user, ready : {
+// RESUME: }, resume : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP2]],
+// RESUME-NEXT: %[[TRUE2:.*]] = cir.const #true
+// RESUME-NEXT: cir.store %[[TRUE2]], %[[FLAG2]]
+// RESUME: cir.call @_ZN2DV12await_resumeE1S(
+// RESUME: },)
+// RESUME: cir.call @resume_use_pair(
+// RESUME-NOEH: } cleanup normal {
+// RESUME-EH: } cleanup all {
+// RESUME: %[[LIVE2:.*]] = cir.load {{.*}} %[[FLAG2]]
+// RESUME-NEXT: cir.if %[[LIVE2]] {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[TMP2]])
+// RESUME-NOEH: } cleanup normal {
+// RESUME-EH: } cleanup all {
+// RESUME: %[[LIVE1:.*]] = cir.load {{.*}} %[[FLAG1]]
+// RESUME-NEXT: cir.if %[[LIVE1]] {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[TMP1]])
+// RESUME: cir.call @mark()
+
+// RESUME-IR-LABEL: define {{.*}} void @two_awaits(
+// RESUME-IR: await.ready:
+// RESUME-IR: {{(call|invoke)}} void @_ZN1SC1Ei(ptr {{.*}} %[[TMP1:[^,]+]], i32 {{.*}} 1)
+// RESUME-IR: {{(call|invoke)}} {{.*}} i32 @_ZN2DA12await_resumeERK1S(
+// RESUME-IR: {{(call|invoke)}} void @_ZN1SC1Ei(ptr {{.*}} %[[TMP2:[^,]+]], i32 {{.*}} 2)
+// RESUME-IR: {{(call|invoke)}} void @resume_use_pair(
+// RESUME-IR: call void @_ZN1SD1Ev(ptr {{.*}} %[[TMP2]])
+// RESUME-IR: call void @_ZN1SD1Ev(ptr {{.*}} %[[TMP1]])
+// RESUME-IR: {{(call|invoke)}} void @mark()
+
+extern "C" Task two_awaits() {
+ resume_use_pair(co_await DA{}, co_await DV{});
+ mark();
+}
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @class_result(
+// RESUME: cir.await(user, ready : {
+// RESUME: cir.call @_ZN2RA12await_resumeERK1S(
+// RESUME: },)
+// RESUME: cir.call @resume_use_result(
+// RESUME: cir.call @_ZN2NTD1Ev(
+// RESUME: cir.call @_ZN1SD1Ev(
+
+// RESUME-IR-LABEL: define {{.*}} void @class_result(
+// RESUME-IR: {{(call|invoke)}} void @_ZN2RA12await_resumeERK1S(
+// RESUME-IR: {{(call|invoke)}} void @resume_use_result(
+// RESUME-IR: call void @_ZN2NTD1Ev(
+// RESUME-IR: call void @_ZN1SD1Ev(
+
+extern "C" Task class_result() {
+ resume_use_result(co_await RA{});
+}
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @in_conditional(
+// RESUME: %[[FLAG:.*]] = cir.alloca "cleanup.cond"
+// RESUME: %[[FALSE:.*]] = cir.const #false
+// RESUME-NEXT: cir.store %[[FALSE]], %[[FLAG]]
+// RESUME: cir.ternary(
+// RESUME: cir.await(user, ready : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP:[^,]+]],
+// RESUME: },)
+// RESUME: cir.call @resume_use(
+// RESUME: %[[LIVE:.*]] = cir.load {{.*}} %[[FLAG]]
+// RESUME-NEXT: cir.if %[[LIVE]] {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[TMP]])
+
+extern "C" Task in_conditional() {
+ resume_use(resume_flag() ? co_await DA{} : 0);
+}
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @agg_init(
+// RESUME: cir.call @_ZN1SC1Ei(
+// RESUME-NEXT: %[[TRUE:.*]] = cir.const #true
+// RESUME-NEXT: cir.store %[[TRUE]], %[[ACTIVE:.*]]
+// RESUME: cir.await(user, ready : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP:[^,]+]],
+// RESUME: cir.call @_ZN2DA12await_resumeERK1S(
+// RESUME: },)
+// RESUME: %[[FALSE:.*]] = cir.const #false
+// RESUME-NEXT: cir.store %[[FALSE]], %[[ACTIVE]]
+// RESUME-NOT: cir.call @_ZN1SD1Ev(%[[TMP]])
+// RESUME: cir.call @resume_next()
+// RESUME: cir.call @resume_use_aggregate(
+// RESUME: cir.call @_ZN3AggD1Ev(
+// RESUME: cir.call @_ZN1SD1Ev(%[[TMP]])
+
+// RESUME-IR-LABEL: define {{.*}} void @agg_init(
+// RESUME-IR: {{(call|invoke)}} {{.*}} i32 @_ZN2DA12await_resumeERK1S(
+// RESUME-IR: {{(call|invoke)}} i32 @resume_next()
+// RESUME-IR: {{(call|invoke)}} void @resume_use_aggregate(
+// RESUME-IR: call void @_ZN3AggD1Ev(
+// RESUME-IR: call void @_ZN1SD1Ev(
+
+extern "C" Task agg_init() {
+ resume_use_aggregate(Agg{S(0), co_await DA{}}, resume_next());
+}
+
+// An unrelated conditional must not create a resume cleanup guard.
+// RESUME-LABEL: cir.func coroutine {{.*}} @agg_plain_resume(
+// RESUME-NOT: cir.alloca "cleanup.cond"
+// RESUME: cir.await(user, ready : {
+// RESUME: cir.call @_ZN2PL12await_resumeEv(
+// RESUME: },)
+// RESUME: cir.call @resume_use_aggregate(
+// RESUME: cir.call @_ZN3AggD1Ev(
+// RESUME: cir.call @mark()
+
+extern "C" Task agg_plain_resume() {
+ resume_use_aggregate(Agg{S(0), co_await PL{}}, resume_flag() ? 1 : 2);
+ mark();
+}
+
+struct E {
+ E(const S & = S(3));
+ ~E();
+};
+
+struct Holder {
+ E e[2];
+ int k;
+};
+
+extern "C" int take(E *);
+extern "C" int take2(Holder *);
+
+struct AN {
+ bool await_ready();
+ void await_suspend(std::coroutine_handle<>);
+ int await_resume(int = take(new E[2]) + take2(new Holder{}));
+};
+
+// Array-element boundaries still apply inside a deferred resume expression.
+// RESUME-LABEL: cir.func coroutine {{.*}} @array_elements(
+// RESUME: cir.await(user, ready : {
+// RESUME: }, resume : {
+// RESUME: cir.do {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP1:[^,]+]],
+// RESUME: cir.call @_ZN1EC1ERK1S(
+// RESUME-NOEH: } cleanup normal {
+// RESUME-EH: } cleanup all {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[TMP1]])
+// RESUME: } while {
+// RESUME: cir.call @take(
+// RESUME: cir.do {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP2:[^,]+]],
+// RESUME: cir.call @_ZN1EC1ERK1S(
+// RESUME-NOEH: } cleanup normal {
+// RESUME-EH: } cleanup all {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[TMP2]])
+// RESUME: } while {
+// RESUME: cir.call @take2(
+// RESUME: cir.call @_ZN2AN12await_resumeEi(
+// RESUME: },)
+// RESUME: cir.call @resume_use(
+
+extern "C" Task array_elements() {
+ resume_use(co_await AN{});
+ mark();
+}
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @asm_operand(
+// RESUME: cir.await(user, ready : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP:[^,]+]],
+// RESUME: },)
+// RESUME: cir.asm(
+// RESUME: cir.call @_ZN1SD1Ev(%[[TMP]])
+// RESUME: cir.call @mark()
+
+extern "C" Task asm_operand() {
+ asm volatile("" : : "r"(co_await DA{}));
+ mark();
+}
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @coyield_arg(
+// RESUME: cir.await(yield, ready : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP:[^,]+]],
+// RESUME: },)
+// RESUME: cir.call @resume_use(
+// RESUME: cir.call @_ZN1SD1Ev(%[[TMP]])
+
+extern "C" Task coyield_arg() {
+ resume_use(co_yield 1);
+}
+
+struct TransformTask {
+ struct promise_type {
+ TransformTask get_return_object();
+ std::suspend_never initial_suspend();
+ std::suspend_never final_suspend() noexcept;
+ void return_void();
+ void unhandled_exception();
+ PL await_transform(int, int = resume_flag() ? resume_arg(S(5)) : 0);
+ };
+};
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @transform_condtemp(
+// RESUME: %[[FLAG:.*]] = cir.alloca "cleanup.cond"
+// RESUME: cir.ternary(
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP:[^,]+]],
+// RESUME: cir.call @_ZN13TransformTask12promise_type15await_transformEii(
+// RESUME: cir.await(user, ready : {
+// RESUME: },)
+// RESUME: cir.call @resume_use(
+// RESUME: %[[LIVE:.*]] = cir.load {{.*}} %[[FLAG]]
+// RESUME-NEXT: cir.if %[[LIVE]] {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[TMP]])
+
+extern "C" TransformTask transform_condtemp() {
+ resume_use(co_await 1);
+}
+
+struct NoexceptInitAwaiter {
+ bool await_ready();
+ void await_suspend(std::coroutine_handle<>);
+ void await_resume(const S & = S(1)) noexcept;
+};
+
+struct NoexceptFinalAwaiter {
+ bool await_ready() noexcept;
+ void await_suspend(std::coroutine_handle<>) noexcept;
+ void await_resume(const S & = S(2)) noexcept;
+};
+
+struct DefaultInitTask {
+ struct promise_type {
+ DefaultInitTask get_return_object();
+ NoexceptInitAwaiter initial_suspend();
+ NoexceptFinalAwaiter final_suspend() noexcept;
+ void return_void();
+ void unhandled_exception();
+ };
+};
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @init_final_noexcept(
+// RESUME-NOT: resume.eh
+// RESUME: cir.await(init, ready : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[INIT_TMP:[^,]+]],
+// RESUME: cir.call @_ZN19NoexceptInitAwaiter12await_resumeERK1S(
+// RESUME: },)
+// RESUME: cir.call @_ZN1SD1Ev(%[[INIT_TMP]])
+// RESUME: }, body : {
+// RESUME: cir.call @mark()
+// RESUME: }, finalSuspend : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[FINAL_TMP:[^,]+]],
+// RESUME: cir.call @_ZN20NoexceptFinalAwaiter12await_resumeERK1S(
+// RESUME: },)
+// RESUME: cir.call @_ZN1SD1Ev(%[[FINAL_TMP]])
+
+extern "C" DefaultInitTask init_final_noexcept() {
+ mark();
+ co_return;
+}
+
+struct InitAwaiter {
+ bool await_ready();
+ void await_suspend(std::coroutine_handle<>);
+ void await_resume(const S & = S(1), int = resume_flag() ? resume_arg(S(2)) : 0);
+};
+
+struct InitTask {
+ struct promise_type {
+ InitTask get_return_object();
+ InitAwaiter initial_suspend();
+ std::suspend_never final_suspend() noexcept;
+ void return_void();
+ void unhandled_exception();
+ };
+};
+
+// RESUME-LABEL: cir.func coroutine {{.*}} @init_try(
+// RESUME-EH: %[[RESUMED:.*]] = cir.alloca "resume.eh"
+// RESUME: cir.await(init, ready : {
+// RESUME-EH: cir.try {
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP1:[^,]+]],
+// RESUME: cir.ternary(
+// RESUME: cir.call @_ZN1SC1Ei(%[[TMP2:[^,]+]],
+// RESUME: cir.call @_ZN11InitAwaiter12await_resumeERK1Si(
+// RESUME: cir.call @_ZN1SD1Ev(%[[TMP2]])
+// RESUME: cir.call @_ZN1SD1Ev(%[[TMP1]])
+// RESUME-EH-NEXT: cir.yield
+// RESUME-EH-NEXT: }
+// RESUME-EH-NEXT: %[[TRUE:.*]] = cir.const #true
+// RESUME-EH-NEXT: cir.store %[[TRUE]], %[[RESUMED]]
+// RESUME-EH: } catch all
+// RESUME: }, body : {
+// RESUME: cir.call @mark()
+
+extern "C" InitTask init_try() {
+ mark();
+ co_return;
+}
+
+struct ReadySuspendArgs {
+ bool await_ready(const S & = S(8));
+ void await_suspend(std::coroutine_handle<>, const S & = S(9));
+ int await_resume();
+};
+
+// Ready and suspend arguments end before the next coroutine phase.
+// RESUME-LABEL: cir.func coroutine {{.*}} @ready_suspend_in_conditional(
+// RESUME: cir.ternary(
+// RESUME: cir.await(user, ready : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[READY:[^,]+]],
+// RESUME: cir.call @_ZN16ReadySuspendArgs11await_readyERK1S(
+// RESUME-NOEH: } cleanup normal {
+// RESUME-EH: } cleanup all {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[READY]])
+// RESUME: }, suspend : {
+// RESUME: cir.call @_ZN1SC1Ei(%[[SUSPEND:[^,]+]],
+// RESUME: cir.call @_ZN16ReadySuspendArgs13await_suspendESt16coroutine_handleIvERK1S(
+// RESUME-NOEH: } cleanup normal {
+// RESUME-EH: } cleanup all {
+// RESUME-NEXT: cir.call @_ZN1SD1Ev(%[[SUSPEND]])
+// RESUME: cir.coro.suspend_point
+// RESUME: }, resume : {
+// RESUME: cir.call @_ZN16ReadySuspendArgs12await_resumeEv(
+
+extern "C" Task ready_suspend_in_conditional(bool b) { resume_use(b ? co_await ReadySuspendArgs{} : 0); }
>From 07656912d297b4a6bee8340aad4d4bd616c6e1a4 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Sat, 24 Oct 2026 12:42:07 +0100
Subject: [PATCH 21/37] [AArch64][GlobalISel] Add deinterleaved store combine.
(#228514)
This adds a combine for converting store(shuffle) into a ST2/ST3/ST4.
The idea is that this can be extended to support interleave /
deinterleave intrinsics in the future once those are supported.
---
llvm/lib/Target/AArch64/AArch64Combine.td | 11 +-
.../GISel/AArch64PreLegalizerCombiner.cpp | 148 +++++++++++++++
.../CodeGen/AArch64/interleaved-accesses.ll | 170 ++++++------------
llvm/test/CodeGen/AArch64/vldn_shuffle.ll | 72 ++------
4 files changed, 234 insertions(+), 167 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index 970b04f0158ce9..e309b18544aa93 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -84,6 +84,14 @@ def deinterleave_load : GICombineRule<
(apply [{ applyDeinterleavingLoad(*${root}, MRI, B, ${matchinfo}); }])
>;
+// Convert store + shuffle to st2/st3/st4
+def interleave_store : GICombineRule<
+ (defs root:$root, register_vector_matchinfo:$matchinfo),
+ (match (G_STORE $value, $addr):$root,
+ [{ return matchInterleavingStore(*${root}, MRI, ${matchinfo}); }]),
+ (apply [{ applyInterleavingStore(*${root}, MRI, B, ${matchinfo}); }])
+>;
+
def AArch64PreLegalizerCombiner: GICombiner<
"AArch64PreLegalizerCombinerImpl", [all_combines,
select_zero_true,
@@ -100,7 +108,8 @@ def AArch64PreLegalizerCombiner: GICombiner<
push_mul_through_sext,
simplify_uaddo,
memcpy_family_combines,
- emit_bzero, deinterleave_load]> {
+ emit_bzero,
+ deinterleave_load, interleave_store]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index 5f4a81011eb2cb..fe3a7193298ff9 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -886,6 +886,154 @@ void applyDeinterleavingLoad(MachineInstr &MI, MachineRegisterInfo &MRI,
MI.eraseFromParent();
}
+// Look for store + shuffles that can be converted to ST2/ST3/ST4
+bool matchInterleavingStore(MachineInstr &MI, MachineRegisterInfo &MRI,
+ SmallVector<Register> &Srcs) {
+ GStore &Store = cast<GStore>(MI);
+ LLT Ty = MRI.getType(Store.getValueReg());
+ if (!Ty.isVector() || !Store.isSimple() ||
+ Store.getMemSizeInBits() != Ty.getSizeInBits() ||
+ !MRI.hasOneNonDBGUse(Store.getValueReg()))
+ return false;
+
+ switch (Ty.getScalarSizeInBits()) {
+ case 8:
+ case 16:
+ case 32:
+ case 64:
+ break;
+ default:
+ return false;
+ }
+
+ MachineInstr *Value = MRI.getVRegDef(Store.getValueReg());
+ if (auto *Shuf = dyn_cast<GShuffleVector>(Value)) {
+ ArrayRef<int> Mask = Shuf->getMask();
+ if (Ty.getSizeInBits() % 128 == 0 &&
+ ShuffleVectorInst::isInterleaveMask(Mask, 2, Mask.size())) {
+ if (MRI.getType(Shuf->getOperand(1).getReg()).getNumElements() ==
+ Mask.size() / 2) {
+ Srcs.push_back(Shuf->getOperand(1).getReg());
+ Srcs.push_back(Shuf->getOperand(2).getReg());
+ return true;
+ }
+
+ MachineInstr *C1 = MRI.getVRegDef(Shuf->getOperand(1).getReg());
+ if (MRI.getType(Shuf->getOperand(1).getReg()).getNumElements() !=
+ Mask.size() ||
+ C1->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
+ C1->getNumOperands() != 3)
+ return false;
+ Srcs.push_back(C1->getOperand(1).getReg());
+ Srcs.push_back(C1->getOperand(2).getReg());
+ return true;
+ }
+ if (Ty.getSizeInBits() % 192 == 0 &&
+ ShuffleVectorInst::isInterleaveMask(Mask, 3, Mask.size())) {
+ MachineInstr *C1 = MRI.getVRegDef(Shuf->getOperand(1).getReg());
+ MachineInstr *C2 = MRI.getVRegDef(Shuf->getOperand(2).getReg());
+ if (C1->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
+ C1->getNumOperands() != 3 ||
+ C2->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
+ C2->getNumOperands() != 3)
+ return false;
+ Srcs.push_back(C1->getOperand(1).getReg());
+ Srcs.push_back(C1->getOperand(2).getReg());
+ Srcs.push_back(C2->getOperand(1).getReg());
+ return true;
+ }
+ if (Ty.getSizeInBits() % 256 == 0 &&
+ ShuffleVectorInst::isInterleaveMask(Mask, 4, Mask.size())) {
+ MachineInstr *C1 = MRI.getVRegDef(Shuf->getOperand(1).getReg());
+ MachineInstr *C2 = MRI.getVRegDef(Shuf->getOperand(2).getReg());
+ if (C1->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
+ C1->getNumOperands() != 3 ||
+ C2->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
+ C2->getNumOperands() != 3)
+ return false;
+ Srcs.push_back(C1->getOperand(1).getReg());
+ Srcs.push_back(C1->getOperand(2).getReg());
+ Srcs.push_back(C2->getOperand(1).getReg());
+ Srcs.push_back(C2->getOperand(2).getReg());
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void applyInterleavingStore(MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &B, SmallVector<Register> &Srcs) {
+ unsigned IID;
+ switch (Srcs.size()) {
+ case 2:
+ IID = Intrinsic::aarch64_neon_st2;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_neon_st3;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_neon_st4;
+ break;
+ default:
+ llvm_unreachable("Unexpected deinterleave size!");
+ }
+
+ // Break down larger vectors into 128 blocks and a possible 64bit remainder
+ LLT Ty = MRI.getType(Srcs[0]);
+ LLT MainTy =
+ LLT::fixed_vector(128 / Ty.getScalarSizeInBits(), Ty.getScalarType());
+ LLT LeftOverTy;
+ SmallVector<SmallVector<Register>> Regs(Srcs.size()), LeftOver(Srcs.size());
+ if (Ty == MainTy) {
+ for (unsigned I = 0; I < Srcs.size(); I++)
+ Regs[I].push_back(Srcs[I]);
+ } else if (Ty == MainTy.divide(2)) {
+ for (unsigned I = 0; I < Srcs.size(); I++)
+ LeftOver[I].push_back(Srcs[I]);
+ LeftOverTy = MainTy.divide(2);
+ } else {
+ for (unsigned I = 0; I < Srcs.size(); I++)
+ extractParts(Srcs[I], Ty, MainTy, LeftOverTy, Regs[I], LeftOver[I], B,
+ MRI);
+ assert(LeftOver[0].size() == 0 ||
+ (LeftOver[0].size() == 1 && LeftOverTy == MainTy.divide(2)));
+ }
+
+ Register Pointer = MI.getOperand(1).getReg();
+ assert(MI.getNumMemOperands() == 1);
+ MachineMemOperand *MMO = *MI.memoperands_begin();
+ MachineFunction &MF = *MI.getParent()->getParent();
+
+ // Generate a ST2/3/4 per vector part
+ auto generateStore = [&](SmallVector<Register> Srcs, LLT Ty,
+ unsigned Offset) {
+ auto PtrOffset = Pointer;
+ if (Offset != 0)
+ PtrOffset =
+ B.buildObjectPtrOffset(MRI.getType(Pointer), Pointer,
+ B.buildConstant(LLT::integer(64), Offset))
+ .getReg(0);
+ auto ST = B.buildIntrinsic(IID, ArrayRef<Register>());
+ for (Register S : Srcs)
+ ST.addReg(S);
+ ST.addReg(PtrOffset);
+ ST.addMemOperand(
+ MF.getMachineMemOperand(MMO, Offset, Ty.multiplyElements(Srcs.size())));
+ };
+ for (unsigned I = 0; I < Regs[0].size(); I++) {
+ for (unsigned S = 0; S < Srcs.size(); S++)
+ Srcs[S] = Regs[S][I];
+ generateStore(Srcs, MainTy, I * 16 * Srcs.size());
+ }
+ if (!LeftOver[0].empty()) {
+ for (unsigned S = 0; S < Srcs.size(); S++)
+ Srcs[S] = LeftOver[S][0];
+ generateStore(Srcs, LeftOverTy, Regs[0].size() * 16 * Srcs.size());
+ }
+ MI.eraseFromParent();
+}
+
class AArch64PreLegalizerCombinerImpl : public Combiner {
protected:
const CombinerHelper Helper;
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index 8198b01faa3782..b47d91fbff8f68 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -519,10 +519,9 @@ define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
;
; CHECK-GI-LABEL: store_factor2:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT: str q0, [x0]
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-GI-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-GI-NEXT: st2 { v0.8b, v1.8b }, [x0]
; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
store <16 x i8> %interleaved.vec, ptr %ptr, align 4
@@ -581,20 +580,10 @@ define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
;
; CHECK-GI-LABEL: store_factor3:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov s3, v0.s[0]
-; CHECK-GI-NEXT: mov s4, v1.s[1]
-; CHECK-GI-NEXT: mov s5, v2.s[2]
-; CHECK-GI-NEXT: mov v3.s[1], v1.s[0]
-; CHECK-GI-NEXT: mov v4.s[1], v2.s[1]
-; CHECK-GI-NEXT: mov v5.s[1], v0.s[3]
-; CHECK-GI-NEXT: mov v3.s[2], v2.s[0]
-; CHECK-GI-NEXT: mov v4.s[2], v0.s[2]
-; CHECK-GI-NEXT: mov v5.s[2], v1.s[3]
-; CHECK-GI-NEXT: mov v3.s[3], v0.s[1]
-; CHECK-GI-NEXT: mov v4.s[3], v1.s[2]
-; CHECK-GI-NEXT: mov v5.s[3], v2.s[3]
-; CHECK-GI-NEXT: stp q3, q4, [x0]
-; CHECK-GI-NEXT: str q5, [x0, #32]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
; CHECK-GI-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -658,24 +647,11 @@ define void @store_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
;
; CHECK-GI-LABEL: store_factor4:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov s4, v0.s[0]
-; CHECK-GI-NEXT: mov s5, v0.s[1]
-; CHECK-GI-NEXT: mov s6, v0.s[2]
-; CHECK-GI-NEXT: mov s0, v0.s[3]
-; CHECK-GI-NEXT: mov v4.s[1], v1.s[0]
-; CHECK-GI-NEXT: mov v5.s[1], v1.s[1]
-; CHECK-GI-NEXT: mov v6.s[1], v1.s[2]
-; CHECK-GI-NEXT: mov v0.s[1], v1.s[3]
-; CHECK-GI-NEXT: mov v4.s[2], v2.s[0]
-; CHECK-GI-NEXT: mov v5.s[2], v2.s[1]
-; CHECK-GI-NEXT: mov v6.s[2], v2.s[2]
-; CHECK-GI-NEXT: mov v0.s[2], v2.s[3]
-; CHECK-GI-NEXT: mov v4.s[3], v3.s[0]
-; CHECK-GI-NEXT: mov v5.s[3], v3.s[1]
-; CHECK-GI-NEXT: mov v6.s[3], v3.s[2]
-; CHECK-GI-NEXT: mov v0.s[3], v3.s[3]
-; CHECK-GI-NEXT: stp q4, q5, [x0]
-; CHECK-GI-NEXT: stp q6, q0, [x0, #32]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
; CHECK-GI-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -859,9 +835,9 @@ define void @store_ptrvec_factor2(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
;
; CHECK-GI-LABEL: store_ptrvec_factor2:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: stp q2, q0, [x0]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: st2 { v0.2d, v1.2d }, [x0]
; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
store <4 x ptr> %interleaved.vec, ptr %ptr, align 4
@@ -895,17 +871,10 @@ define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
;
; CHECK-GI-LABEL: store_ptrvec_factor3:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: zip1 v3.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: zip2 v1.2d, v1.2d, v2.2d
-; CHECK-GI-NEXT: mov v2.d[1], v0.d[1]
-; CHECK-GI-NEXT: mov x8, v3.d[1]
-; CHECK-GI-NEXT: mov x9, v2.d[1]
-; CHECK-GI-NEXT: mov x10, v1.d[1]
-; CHECK-GI-NEXT: mov v3.d[1], x8
-; CHECK-GI-NEXT: mov v2.d[1], x9
-; CHECK-GI-NEXT: mov v1.d[1], x10
-; CHECK-GI-NEXT: stp q3, q2, [x0]
-; CHECK-GI-NEXT: str q1, [x0, #32]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
; CHECK-GI-NEXT: ret
%s0 = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%s1 = shufflevector <2 x ptr> %v2, <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
@@ -946,12 +915,11 @@ define void @store_ptrvec_factor4(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
;
; CHECK-GI-LABEL: store_ptrvec_factor4:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: zip1 v4.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: zip1 v5.2d, v2.2d, v3.2d
-; CHECK-GI-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: zip2 v1.2d, v2.2d, v3.2d
-; CHECK-GI-NEXT: stp q4, q5, [x0]
-; CHECK-GI-NEXT: stp q0, q1, [x0, #32]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
; CHECK-GI-NEXT: ret
%s0 = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%s1 = shufflevector <2 x ptr> %v2, <2 x ptr> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -1160,8 +1128,9 @@ define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
;
; CHECK-GI-LABEL: store_undef_mask_factor2:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: str q0, [x0, #16]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: st2 { v0.4s, v1.4s }, [x0]
; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 6, i32 3, i32 7>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
@@ -1216,21 +1185,10 @@ define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
;
; CHECK-GI-LABEL: store_undef_mask_factor3:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: trn1 v3.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: mov s5, v2.s[2]
-; CHECK-GI-NEXT: mov v6.s[1], v2.s[1]
-; CHECK-GI-NEXT: mov v3.d[1], v0.d[0]
-; CHECK-GI-NEXT: mov v5.s[1], v0.s[3]
-; CHECK-GI-NEXT: mov v6.s[2], v0.s[2]
-; CHECK-GI-NEXT: mov s4, v3.s[0]
-; CHECK-GI-NEXT: mov v5.s[2], v1.s[3]
-; CHECK-GI-NEXT: mov v6.s[3], v1.s[2]
-; CHECK-GI-NEXT: mov v4.s[1], v3.s[1]
-; CHECK-GI-NEXT: mov v5.s[3], v2.s[3]
-; CHECK-GI-NEXT: mov v4.s[2], v3.s[2]
-; CHECK-GI-NEXT: str q5, [x0, #32]
-; CHECK-GI-NEXT: mov v4.s[3], v3.s[3]
-; CHECK-GI-NEXT: stp q4, q6, [x0]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
; CHECK-GI-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -1289,22 +1247,11 @@ define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
;
; CHECK-GI-LABEL: store_undef_mask_factor4:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov s4, v0.s[2]
-; CHECK-GI-NEXT: mov s5, v0.s[3]
-; CHECK-GI-NEXT: mov s0, v0.s[0]
-; CHECK-GI-NEXT: mov v6.s[1], v1.s[1]
-; CHECK-GI-NEXT: mov v4.s[1], v1.s[2]
-; CHECK-GI-NEXT: mov v5.s[1], v1.s[3]
-; CHECK-GI-NEXT: mov v0.s[1], v1.s[0]
-; CHECK-GI-NEXT: mov v6.s[2], v2.s[1]
-; CHECK-GI-NEXT: mov v4.s[2], v2.s[2]
-; CHECK-GI-NEXT: mov v5.s[2], v2.s[3]
-; CHECK-GI-NEXT: mov v0.s[2], v2.s[0]
-; CHECK-GI-NEXT: mov v6.s[3], v3.s[1]
-; CHECK-GI-NEXT: mov v4.s[3], v3.s[2]
-; CHECK-GI-NEXT: mov v5.s[3], v3.s[3]
-; CHECK-GI-NEXT: stp q0, q6, [x0]
-; CHECK-GI-NEXT: stp q4, q5, [x0, #32]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
; CHECK-GI-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -2587,12 +2534,13 @@ define void @store_factor2_wide2(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
;
; CHECK-GI-LABEL: store_factor2_wide2:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: zip1 v4.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: zip1 v2.4s, v1.4s, v3.4s
-; CHECK-GI-NEXT: zip2 v1.4s, v1.4s, v3.4s
-; CHECK-GI-NEXT: stp q4, q0, [x0]
-; CHECK-GI-NEXT: stp q2, q1, [x0, #32]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: mov v4.16b, v1.16b
+; CHECK-GI-NEXT: add x8, x0, #32
+; CHECK-GI-NEXT: mov v1.16b, v2.16b
+; CHECK-GI-NEXT: mov v5.16b, v3.16b
+; CHECK-GI-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-GI-NEXT: st2 { v4.4s, v5.4s }, [x8]
; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
store <16 x i32> %interleaved.vec, ptr %ptr, align 4
@@ -3213,10 +3161,9 @@ define void @store_factor2_intrinsic(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
;
; CHECK-GI-LABEL: store_factor2_intrinsic:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT: str q0, [x0]
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-GI-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-GI-NEXT: st2 { v0.8b, v1.8b }, [x0]
; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> %v0, <8 x i8> %v1)
store <16 x i8> %interleaved.intrinsic, ptr %ptr, align 4
@@ -3257,12 +3204,13 @@ define void @store_factor2_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1
;
; CHECK-GI-LABEL: store_factor2_wide_intrinsic:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: zip1 v4.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: zip1 v2.4s, v1.4s, v3.4s
-; CHECK-GI-NEXT: zip2 v1.4s, v1.4s, v3.4s
-; CHECK-GI-NEXT: stp q4, q0, [x0]
-; CHECK-GI-NEXT: stp q2, q1, [x0, #32]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: mov v4.16b, v1.16b
+; CHECK-GI-NEXT: add x8, x0, #32
+; CHECK-GI-NEXT: mov v1.16b, v2.16b
+; CHECK-GI-NEXT: mov v5.16b, v3.16b
+; CHECK-GI-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-GI-NEXT: st2 { v4.4s, v5.4s }, [x8]
; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v0, <8 x i32> %v1)
store <16 x i32> %interleaved.intrinsic, ptr %ptr, align 4
@@ -3930,9 +3878,9 @@ define void @store_ptrvec_factor2_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %
;
; CHECK-GI-LABEL: store_ptrvec_factor2_intrinsic:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: stp q2, q0, [x0]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: st2 { v0.2d, v1.2d }, [x0]
; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> %v0, <2 x ptr> %v1)
store <4 x ptr> %interleaved.intrinsic, ptr %ptr, align 4
@@ -4019,9 +3967,9 @@ define void @store_undef_mask_factor2_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i3
;
; CHECK-GI-LABEL: store_undef_mask_factor2_intrinsic:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: zip1 v2.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: stp q2, q0, [x0]
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: st2 { v0.4s, v1.4s }, [x0]
; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
store <8 x i32> %interleaved.intrinsic, ptr %ptr, align 4
diff --git a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
index 9f18a44003131c..cfe7688011ed2b 100644
--- a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
@@ -293,10 +293,8 @@ define void @vld4(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32
; CHECK-GI-NEXT: mov v4.s[3], v1.s[2]
; CHECK-GI-NEXT: mov v0.s[3], v1.s[3]
; CHECK-GI-NEXT: fadd v1.4s, v3.4s, v2.4s
-; CHECK-GI-NEXT: fadd v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT: zip1 v2.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: zip2 v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: stp q2, q0, [x9]
+; CHECK-GI-NEXT: fadd v2.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: st2 { v1.4s, v2.4s }, [x9]
; CHECK-GI-NEXT: b.ne .LBB2_1
; CHECK-GI-NEXT: // %bb.2: // %while.end
; CHECK-GI-NEXT: ret
@@ -698,10 +696,8 @@ define void @vld4_multiuse(ptr nocapture readonly %pSrc, ptr noalias nocapture %
; CHECK-GI-NEXT: mov v4.s[3], v1.s[2]
; CHECK-GI-NEXT: mov v0.s[3], v1.s[3]
; CHECK-GI-NEXT: fadd v1.4s, v3.4s, v2.4s
-; CHECK-GI-NEXT: fadd v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT: zip1 v2.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: zip2 v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: stp q2, q0, [x9]
+; CHECK-GI-NEXT: fadd v2.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: st2 { v1.4s, v2.4s }, [x9]
; CHECK-GI-NEXT: b.ne .LBB6_1
; CHECK-GI-NEXT: // %bb.2: // %while.end
; CHECK-GI-NEXT: ret
@@ -1269,10 +1265,8 @@ define void @store_factor2(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1) {
; CHECK-GI-NEXT: .cfi_startproc
; CHECK-GI-NEXT: // %bb.0:
; CHECK-GI-NEXT: trn1 v2.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: trn1 v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: zip1 v1.4s, v2.4s, v0.4s
-; CHECK-GI-NEXT: zip2 v0.4s, v2.4s, v0.4s
-; CHECK-GI-NEXT: stp q1, q0, [x0]
+; CHECK-GI-NEXT: trn1 v3.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: st2 { v2.4s, v3.4s }, [x0]
; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -1384,28 +1378,15 @@ define void @store_factor3(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2
; CHECK-GI-NEXT: .cfi_startproc
; CHECK-GI-NEXT: // %bb.0:
; CHECK-GI-NEXT: ext v3.16b, v0.16b, v1.16b, #12
-; CHECK-GI-NEXT: ext v4.16b, v1.16b, v2.16b, #12
-; CHECK-GI-NEXT: ext v5.16b, v2.16b, v0.16b, #12
+; CHECK-GI-NEXT: ext v6.16b, v1.16b, v2.16b, #12
; CHECK-GI-NEXT: zip2 v3.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT: zip2 v4.4s, v1.4s, v4.4s
-; CHECK-GI-NEXT: zip2 v5.4s, v2.4s, v5.4s
; CHECK-GI-NEXT: mov v3.s[0], v0.s[0]
+; CHECK-GI-NEXT: ext v0.16b, v2.16b, v0.16b, #12
+; CHECK-GI-NEXT: zip2 v4.4s, v1.4s, v6.4s
; CHECK-GI-NEXT: mov v4.s[0], v1.s[0]
+; CHECK-GI-NEXT: zip2 v5.4s, v2.4s, v0.4s
; CHECK-GI-NEXT: mov v5.s[0], v2.s[0]
-; CHECK-GI-NEXT: mov s0, v3.s[0]
-; CHECK-GI-NEXT: mov s1, v4.s[1]
-; CHECK-GI-NEXT: mov s2, v5.s[2]
-; CHECK-GI-NEXT: mov v0.s[1], v4.s[0]
-; CHECK-GI-NEXT: mov v1.s[1], v5.s[1]
-; CHECK-GI-NEXT: mov v2.s[1], v3.s[3]
-; CHECK-GI-NEXT: mov v0.s[2], v5.s[0]
-; CHECK-GI-NEXT: mov v1.s[2], v3.s[2]
-; CHECK-GI-NEXT: mov v2.s[2], v4.s[3]
-; CHECK-GI-NEXT: mov v0.s[3], v3.s[1]
-; CHECK-GI-NEXT: mov v1.s[3], v4.s[2]
-; CHECK-GI-NEXT: mov v2.s[3], v5.s[3]
-; CHECK-GI-NEXT: stp q0, q1, [x0]
-; CHECK-GI-NEXT: str q2, [x0, #32]
+; CHECK-GI-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
@@ -1456,27 +1437,10 @@ define void @store_factor4(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2
; CHECK-GI-NEXT: .cfi_startproc
; CHECK-GI-NEXT: // %bb.0:
; CHECK-GI-NEXT: trn1 v4.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: trn1 v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: trn1 v2.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: trn1 v0.4s, v3.4s, v0.4s
-; CHECK-GI-NEXT: mov s5, v4.s[0]
-; CHECK-GI-NEXT: mov s6, v4.s[1]
-; CHECK-GI-NEXT: mov s7, v4.s[2]
-; CHECK-GI-NEXT: mov s4, v4.s[3]
-; CHECK-GI-NEXT: mov v5.s[1], v1.s[0]
-; CHECK-GI-NEXT: mov v6.s[1], v1.s[1]
-; CHECK-GI-NEXT: mov v7.s[1], v1.s[2]
-; CHECK-GI-NEXT: mov v4.s[1], v1.s[3]
-; CHECK-GI-NEXT: mov v5.s[2], v2.s[0]
-; CHECK-GI-NEXT: mov v6.s[2], v2.s[1]
-; CHECK-GI-NEXT: mov v7.s[2], v2.s[2]
-; CHECK-GI-NEXT: mov v4.s[2], v2.s[3]
-; CHECK-GI-NEXT: mov v5.s[3], v0.s[0]
-; CHECK-GI-NEXT: mov v6.s[3], v0.s[1]
-; CHECK-GI-NEXT: mov v7.s[3], v0.s[2]
-; CHECK-GI-NEXT: mov v4.s[3], v0.s[3]
-; CHECK-GI-NEXT: stp q5, q6, [x0]
-; CHECK-GI-NEXT: stp q7, q4, [x0, #32]
+; CHECK-GI-NEXT: trn1 v5.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: trn1 v7.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -2304,10 +2268,8 @@ define void @store_factor2_intrinsic(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1) {
; CHECK-GI-NEXT: .cfi_startproc
; CHECK-GI-NEXT: // %bb.0:
; CHECK-GI-NEXT: trn1 v2.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: trn1 v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: zip1 v1.4s, v2.4s, v0.4s
-; CHECK-GI-NEXT: zip2 v0.4s, v2.4s, v0.4s
-; CHECK-GI-NEXT: stp q1, q0, [x0]
+; CHECK-GI-NEXT: trn1 v3.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: st2 { v2.4s, v3.4s }, [x0]
; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
>From 4b0b58f21ab623a983ab295e82d525fa5ba2ba5e Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 24 Oct 2026 07:45:51 -0400
Subject: [PATCH 22/37] [SLP][NFC]Add a test with incorrect vectorization, NFC
(#235277)
---
.../X86/minbw-alt-cast-demoted-operand.ll | 246 ++++++++++++++++++
1 file changed, 246 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll
new file mode 100644
index 00000000000000..b45037e1be67a1
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll
@@ -0,0 +1,246 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @alt_zext_sext_unsigned_demoted_operand(ptr noalias %p, ptr noalias %q) {
+; CHECK-LABEL: define void @alt_zext_sext_unsigned_demoted_operand(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0
+; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <4 x i8> [[TMP0]], splat (i8 -6)
+; CHECK-NEXT: [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i32> [[TMP6]], splat (i32 16843009)
+; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[PQ0]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr %p, i64 0
+ %a0 = load i8, ptr %pa0, align 1
+ %pa1 = getelementptr inbounds i8, ptr %p, i64 1
+ %a1 = load i8, ptr %pa1, align 1
+ %pa2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a2 = load i8, ptr %pa2, align 1
+ %pa3 = getelementptr inbounds i8, ptr %p, i64 3
+ %a3 = load i8, ptr %pa3, align 1
+ %s0 = zext i8 %a0 to i16
+ %z0 = zext i16 %s0 to i32
+ %c0 = icmp ult i16 %s0, 250
+ %m0 = sext i1 %c0 to i32
+ %n0 = and i32 %z0, %m0
+ %r0 = mul i32 %n0, 16843009
+ %pq0 = getelementptr inbounds i32, ptr %q, i64 0
+ store i32 %r0, ptr %pq0, align 4
+ %s1 = zext i8 %a1 to i16
+ %z1 = sext i16 %s1 to i32
+ %c1 = icmp ult i16 %s1, 250
+ %m1 = sext i1 %c1 to i32
+ %n1 = and i32 %z1, %m1
+ %r1 = mul i32 %n1, 16843009
+ %pq1 = getelementptr inbounds i32, ptr %q, i64 1
+ store i32 %r1, ptr %pq1, align 4
+ %s2 = zext i8 %a2 to i16
+ %z2 = zext i16 %s2 to i32
+ %c2 = icmp ult i16 %s2, 250
+ %m2 = sext i1 %c2 to i32
+ %n2 = and i32 %z2, %m2
+ %r2 = mul i32 %n2, 16843009
+ %pq2 = getelementptr inbounds i32, ptr %q, i64 2
+ store i32 %r2, ptr %pq2, align 4
+ %s3 = zext i8 %a3 to i16
+ %z3 = sext i16 %s3 to i32
+ %c3 = icmp ult i16 %s3, 250
+ %m3 = sext i1 %c3 to i32
+ %n3 = and i32 %z3, %m3
+ %r3 = mul i32 %n3, 16843009
+ %pq3 = getelementptr inbounds i32, ptr %q, i64 3
+ store i32 %r3, ptr %pq3, align 4
+ ret void
+}
+
+define void @alt_sext_zext_signed_demoted_operand(ptr noalias %p, ptr noalias %q) {
+; CHECK-LABEL: define void @alt_sext_zext_signed_demoted_operand(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0
+; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <4 x i8> [[TMP0]], splat (i8 100)
+; CHECK-NEXT: [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i32> [[TMP6]], splat (i32 16843009)
+; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[PQ0]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr %p, i64 0
+ %a0 = load i8, ptr %pa0, align 1
+ %pa1 = getelementptr inbounds i8, ptr %p, i64 1
+ %a1 = load i8, ptr %pa1, align 1
+ %pa2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a2 = load i8, ptr %pa2, align 1
+ %pa3 = getelementptr inbounds i8, ptr %p, i64 3
+ %a3 = load i8, ptr %pa3, align 1
+ %s0 = sext i8 %a0 to i16
+ %z0 = sext i16 %s0 to i32
+ %c0 = icmp slt i16 %s0, 100
+ %m0 = sext i1 %c0 to i32
+ %n0 = and i32 %z0, %m0
+ %r0 = mul i32 %n0, 16843009
+ %pq0 = getelementptr inbounds i32, ptr %q, i64 0
+ store i32 %r0, ptr %pq0, align 4
+ %s1 = sext i8 %a1 to i16
+ %z1 = zext i16 %s1 to i32
+ %c1 = icmp slt i16 %s1, 100
+ %m1 = sext i1 %c1 to i32
+ %n1 = and i32 %z1, %m1
+ %r1 = mul i32 %n1, 16843009
+ %pq1 = getelementptr inbounds i32, ptr %q, i64 1
+ store i32 %r1, ptr %pq1, align 4
+ %s2 = sext i8 %a2 to i16
+ %z2 = sext i16 %s2 to i32
+ %c2 = icmp slt i16 %s2, 100
+ %m2 = sext i1 %c2 to i32
+ %n2 = and i32 %z2, %m2
+ %r2 = mul i32 %n2, 16843009
+ %pq2 = getelementptr inbounds i32, ptr %q, i64 2
+ store i32 %r2, ptr %pq2, align 4
+ %s3 = sext i8 %a3 to i16
+ %z3 = zext i16 %s3 to i32
+ %c3 = icmp slt i16 %s3, 100
+ %m3 = sext i1 %c3 to i32
+ %n3 = and i32 %z3, %m3
+ %r3 = mul i32 %n3, 16843009
+ %pq3 = getelementptr inbounds i32, ptr %q, i64 3
+ store i32 %r3, ptr %pq3, align 4
+ ret void
+}
+
+define void @alt_uitofp_sitofp_unsigned_demoted_operand(ptr noalias %p, ptr noalias %q) {
+; CHECK-LABEL: define void @alt_uitofp_sitofp_unsigned_demoted_operand(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0
+; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = uitofp <4 x i8> [[TMP0]] to <4 x float>
+; CHECK-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP0]] to <4 x float>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <4 x i8> [[TMP0]], splat (i8 -6)
+; CHECK-NEXT: [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = and <4 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[PQ0]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr %p, i64 0
+ %a0 = load i8, ptr %pa0, align 1
+ %pa1 = getelementptr inbounds i8, ptr %p, i64 1
+ %a1 = load i8, ptr %pa1, align 1
+ %pa2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a2 = load i8, ptr %pa2, align 1
+ %pa3 = getelementptr inbounds i8, ptr %p, i64 3
+ %a3 = load i8, ptr %pa3, align 1
+ %s0 = zext i8 %a0 to i16
+ %f0 = uitofp i16 %s0 to float
+ %z0 = bitcast float %f0 to i32
+ %c0 = icmp ult i16 %s0, 250
+ %m0 = sext i1 %c0 to i32
+ %n0 = and i32 %z0, %m0
+ %pq0 = getelementptr inbounds i32, ptr %q, i64 0
+ store i32 %n0, ptr %pq0, align 4
+ %s1 = zext i8 %a1 to i16
+ %f1 = sitofp i16 %s1 to float
+ %z1 = bitcast float %f1 to i32
+ %c1 = icmp ult i16 %s1, 250
+ %m1 = sext i1 %c1 to i32
+ %n1 = and i32 %z1, %m1
+ %pq1 = getelementptr inbounds i32, ptr %q, i64 1
+ store i32 %n1, ptr %pq1, align 4
+ %s2 = zext i8 %a2 to i16
+ %f2 = uitofp i16 %s2 to float
+ %z2 = bitcast float %f2 to i32
+ %c2 = icmp ult i16 %s2, 250
+ %m2 = sext i1 %c2 to i32
+ %n2 = and i32 %z2, %m2
+ %pq2 = getelementptr inbounds i32, ptr %q, i64 2
+ store i32 %n2, ptr %pq2, align 4
+ %s3 = zext i8 %a3 to i16
+ %f3 = sitofp i16 %s3 to float
+ %z3 = bitcast float %f3 to i32
+ %c3 = icmp ult i16 %s3, 250
+ %m3 = sext i1 %c3 to i32
+ %n3 = and i32 %z3, %m3
+ %pq3 = getelementptr inbounds i32, ptr %q, i64 3
+ store i32 %n3, ptr %pq3, align 4
+ ret void
+}
+
+define void @alt_sitofp_uitofp_signed_demoted_operand(ptr noalias %p, ptr noalias %q) {
+; CHECK-LABEL: define void @alt_sitofp_uitofp_signed_demoted_operand(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0
+; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i8> [[TMP0]] to <4 x float>
+; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i8> [[TMP0]] to <4 x float>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = icmp slt <4 x i8> [[TMP0]], splat (i8 100)
+; CHECK-NEXT: [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = and <4 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[PQ0]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr %p, i64 0
+ %a0 = load i8, ptr %pa0, align 1
+ %pa1 = getelementptr inbounds i8, ptr %p, i64 1
+ %a1 = load i8, ptr %pa1, align 1
+ %pa2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a2 = load i8, ptr %pa2, align 1
+ %pa3 = getelementptr inbounds i8, ptr %p, i64 3
+ %a3 = load i8, ptr %pa3, align 1
+ %s0 = sext i8 %a0 to i16
+ %f0 = sitofp i16 %s0 to float
+ %z0 = bitcast float %f0 to i32
+ %c0 = icmp slt i16 %s0, 100
+ %m0 = sext i1 %c0 to i32
+ %n0 = and i32 %z0, %m0
+ %pq0 = getelementptr inbounds i32, ptr %q, i64 0
+ store i32 %n0, ptr %pq0, align 4
+ %s1 = sext i8 %a1 to i16
+ %f1 = uitofp i16 %s1 to float
+ %z1 = bitcast float %f1 to i32
+ %c1 = icmp slt i16 %s1, 100
+ %m1 = sext i1 %c1 to i32
+ %n1 = and i32 %z1, %m1
+ %pq1 = getelementptr inbounds i32, ptr %q, i64 1
+ store i32 %n1, ptr %pq1, align 4
+ %s2 = sext i8 %a2 to i16
+ %f2 = sitofp i16 %s2 to float
+ %z2 = bitcast float %f2 to i32
+ %c2 = icmp slt i16 %s2, 100
+ %m2 = sext i1 %c2 to i32
+ %n2 = and i32 %z2, %m2
+ %pq2 = getelementptr inbounds i32, ptr %q, i64 2
+ store i32 %n2, ptr %pq2, align 4
+ %s3 = sext i8 %a3 to i16
+ %f3 = uitofp i16 %s3 to float
+ %z3 = bitcast float %f3 to i32
+ %c3 = icmp slt i16 %s3, 100
+ %m3 = sext i1 %c3 to i32
+ %n3 = and i32 %z3, %m3
+ %pq3 = getelementptr inbounds i32, ptr %q, i64 3
+ store i32 %n3, ptr %pq3, align 4
+ ret void
+}
>From a58a22722053b88bf8a3af939712bdacd2f0a01d Mon Sep 17 00:00:00 2001
From: Petar Jovanovic <pjovanov at amd.com>
Date: Sat, 24 Oct 2026 13:47:41 +0200
Subject: [PATCH 23/37] [CodeGen] Drop dead SlotIndexes before allocation
(#224031)
SlotIndexes keeps the index list entry of an erased instruction and only
clears its instruction pointer. Live range sizes are measured in slot
indexes and greedy ranks ranges by size, so the leftovers inflate some
ranges more than others and reorder allocation, spilling heavily on
register-starved functions.
Add SlotIndexes::compactIndexes() to erase them. Erased entries are
unlinked, so holding an index to one is now a bug; linkedEntry() asserts
on it. The one analysis that did was LiveIntervals, so add
removeUnreferencedIntervals() to retire the intervals responsible.
Off by default behind -greedy-compact-slot-indexes, since it changes
allocation across much of the test suite.
The test file greedy-compact-slot-indexes-priority.mir by Q. Colombet.
---
llvm/include/llvm/CodeGen/LiveIntervals.h | 4 +
llvm/include/llvm/CodeGen/SlotIndexes.h | 29 +-
llvm/lib/CodeGen/LiveIntervals.cpp | 17 ++
llvm/lib/CodeGen/RegAllocGreedy.cpp | 13 +-
llvm/lib/CodeGen/SlotIndexes.cpp | 43 +++
.../greedy-compact-slot-indexes-priority.mir | 82 +++++
.../AMDGPU/greedy-compact-slot-indexes.ll | 35 +++
llvm/unittests/CodeGen/SlotIndexesTest.cpp | 288 ++++++++++++++++++
8 files changed, 504 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/greedy-compact-slot-indexes-priority.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/greedy-compact-slot-indexes.ll
diff --git a/llvm/include/llvm/CodeGen/LiveIntervals.h b/llvm/include/llvm/CodeGen/LiveIntervals.h
index 966c38692d0048..b1da7ab5e7dee8 100644
--- a/llvm/include/llvm/CodeGen/LiveIntervals.h
+++ b/llvm/include/llvm/CodeGen/LiveIntervals.h
@@ -252,6 +252,10 @@ class LiveIntervals {
SlotIndexes *getSlotIndexes() const { return Indexes; }
+ /// Remove intervals for virtual registers no instruction refers to any
+ /// more. Their segments sit on erased indexes. Returns the number removed.
+ LLVM_ABI unsigned removeUnreferencedIntervals();
+
/// Returns true if the specified machine instr has been removed or was
/// never entered in the map.
bool isNotInMIMap(const MachineInstr &Instr) const {
diff --git a/llvm/include/llvm/CodeGen/SlotIndexes.h b/llvm/include/llvm/CodeGen/SlotIndexes.h
index a5f397fd8626b4..78873c153b47fa 100644
--- a/llvm/include/llvm/CodeGen/SlotIndexes.h
+++ b/llvm/include/llvm/CodeGen/SlotIndexes.h
@@ -47,10 +47,14 @@ class raw_ostream;
class IndexListEntry : public ilist_node<IndexListEntry> {
MachineInstr *mi;
unsigned index;
+ bool erased = false;
public:
IndexListEntry(MachineInstr *mi, unsigned index) : mi(mi), index(index) {}
+ bool isErased() const { return erased; }
+ void setErased() { erased = true; }
+
MachineInstr* getInstr() const { return mi; }
void setInstr(MachineInstr *mi) {
this->mi = mi;
@@ -99,6 +103,14 @@ class raw_ostream;
return lie.getPointer();
}
+ /// For operations that cannot use an erased entry. Comparison still can.
+ IndexListEntry *linkedEntry() const {
+ assert(!listEntry()->isErased() &&
+ "SlotIndex outlived the instruction it pointed at; the analysis "
+ "holding it must retire it when the instruction is erased");
+ return listEntry();
+ }
+
unsigned getIndex() const {
return listEntry()->getIndex() | getSlot();
}
@@ -250,7 +262,7 @@ class raw_ostream;
SlotIndex getNextSlot() const {
Slot s = getSlot();
if (s == Slot_Dead) {
- return SlotIndex(&*++listEntry()->getIterator(), Slot_Block);
+ return SlotIndex(&*++linkedEntry()->getIterator(), Slot_Block);
}
return SlotIndex(listEntry(), s + 1);
}
@@ -258,7 +270,7 @@ class raw_ostream;
/// Returns the next index. This is the index corresponding to the this
/// index's slot, but for the next instruction.
SlotIndex getNextIndex() const {
- return SlotIndex(&*++listEntry()->getIterator(), getSlot());
+ return SlotIndex(&*++linkedEntry()->getIterator(), getSlot());
}
/// Returns the previous slot in the index list. This could be either the
@@ -270,7 +282,7 @@ class raw_ostream;
SlotIndex getPrevSlot() const {
Slot s = getSlot();
if (s == Slot_Block) {
- return SlotIndex(&*--listEntry()->getIterator(), Slot_Dead);
+ return SlotIndex(&*--linkedEntry()->getIterator(), Slot_Dead);
}
return SlotIndex(listEntry(), s - 1);
}
@@ -278,7 +290,7 @@ class raw_ostream;
/// Returns the previous index. This is the index corresponding to this
/// index's slot, but for the previous instruction.
SlotIndex getPrevIndex() const {
- return SlotIndex(&*--listEntry()->getIterator(), getSlot());
+ return SlotIndex(&*--linkedEntry()->getIterator(), getSlot());
}
};
@@ -393,7 +405,7 @@ class raw_ostream;
/// Returns the instruction for the given index, or null if the given
/// index has no instruction associated with it.
MachineInstr* getInstructionFromIndex(SlotIndex index) const {
- return index.listEntry()->getInstr();
+ return index.linkedEntry()->getInstr();
}
/// Returns true if \p Idx refers to an entry created to mark a basic block
@@ -413,7 +425,7 @@ class raw_ostream;
/// Returns the next non-null index, if one exists.
/// Otherwise returns getLastIndex().
SlotIndex getNextNonNullIndex(SlotIndex Index) {
- IndexList::iterator I = Index.listEntry()->getIterator();
+ IndexList::iterator I = Index.linkedEntry()->getIterator();
IndexList::iterator E = indexList.end();
while (++I != E)
if (I->getInstr())
@@ -656,6 +668,11 @@ class raw_ostream;
/// Renumber all indexes using the default instruction distance.
LLVM_ABI void packIndexes();
+
+ /// Erase index list entries left behind by deleted instructions, then
+ /// renumber. Erased entries are unlinked, so nothing may hold one.
+ /// Returns the number erased.
+ LLVM_ABI unsigned compactIndexes();
};
// Specialize IntervalMapInfo for half-open slot index intervals.
diff --git a/llvm/lib/CodeGen/LiveIntervals.cpp b/llvm/lib/CodeGen/LiveIntervals.cpp
index e603bd1f61e455..538fcdf80a1c67 100644
--- a/llvm/lib/CodeGen/LiveIntervals.cpp
+++ b/llvm/lib/CodeGen/LiveIntervals.cpp
@@ -18,6 +18,7 @@
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/CodeGen/LiveInterval.h"
#include "llvm/CodeGen/LiveIntervalCalc.h"
@@ -59,6 +60,9 @@ using namespace llvm;
#define DEBUG_TYPE "regalloc"
+STATISTIC(NumUnreferencedIntervals,
+ "Number of intervals removed for unreferenced virtual registers");
+
AnalysisKey LiveIntervalsAnalysis::Key;
LiveIntervalsAnalysis::Result
@@ -178,6 +182,19 @@ void LiveIntervals::analyze(MachineFunction &fn) {
}
}
+unsigned LiveIntervals::removeUnreferencedIntervals() {
+ unsigned Removed = 0;
+ for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
+ Register Reg = Register::index2VirtReg(I);
+ if (hasInterval(Reg) && MRI->reg_empty(Reg)) {
+ removeInterval(Reg);
+ ++Removed;
+ }
+ }
+ NumUnreferencedIntervals += Removed;
+ return Removed;
+}
+
void LiveIntervals::print(raw_ostream &OS) const {
OS << "********** INTERVALS **********\n";
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 99648f9fd15767..ec00926f139462 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -109,6 +109,11 @@ static cl::opt<bool> ExhaustiveSearch(
"and interference cutoffs of last chance recoloring"),
cl::Hidden);
+static cl::opt<bool> CompactSlotIndexes(
+ "greedy-compact-slot-indexes", cl::Hidden, cl::init(false),
+ cl::desc("Drop SlotIndexes entries for erased instructions before "
+ "allocating, so live range sizes reflect real code distance"));
+
// This option should be deprecated!
// FIXME: Find a good default for this flag and remove the flag.
static cl::opt<unsigned>
@@ -3077,7 +3082,13 @@ bool RAGreedy::run(MachineFunction &mf) {
// Renumber to get accurate and consistent results from
// SlotIndexes::getApproxInstrDistance.
- Indexes->packIndexes();
+ if (CompactSlotIndexes) {
+ // Compaction unlinks what it erases; this is what makes that safe.
+ LIS->removeUnreferencedIntervals();
+ Indexes->compactIndexes();
+ } else {
+ Indexes->packIndexes();
+ }
initializeCSRCost();
diff --git a/llvm/lib/CodeGen/SlotIndexes.cpp b/llvm/lib/CodeGen/SlotIndexes.cpp
index b141cd1c8aa271..f3031c54d7faa1 100644
--- a/llvm/lib/CodeGen/SlotIndexes.cpp
+++ b/llvm/lib/CodeGen/SlotIndexes.cpp
@@ -46,6 +46,8 @@ INITIALIZE_PASS(SlotIndexesWrapperPass, DEBUG_TYPE, "Slot index numbering",
false, false)
STATISTIC(NumLocalRenum, "Number of local renumberings");
+STATISTIC(NumDeadEntriesErased,
+ "Number of index list entries erased for deleted instructions");
void SlotIndexesWrapperPass::getAnalysisUsage(AnalysisUsage &au) const {
au.setPreservesAll();
@@ -322,6 +324,47 @@ void SlotIndexes::packIndexes() {
Entry.setIndex(Index * SlotIndex::InstrDist);
}
+unsigned SlotIndexes::compactIndexes() {
+ if (indexList.empty())
+ return 0;
+
+ SmallPtrSet<const IndexListEntry *, 32> Keep;
+
+ // The first and last entries anchor getZeroIndex() and getLastIndex().
+ Keep.insert(&indexList.front());
+ Keep.insert(&indexList.back());
+
+ for (const auto &Range : MBBRanges) {
+ if (Range.first.isValid())
+ Keep.insert(Range.first.listEntry());
+ if (Range.second.isValid())
+ Keep.insert(Range.second.listEntry());
+ }
+
+ // Entries must be unlinked, not zero-width: that would break getNextSlot().
+ SmallVector<IndexListEntry *, 16> Erased;
+ unsigned Index = 0;
+ for (auto It = indexList.begin(), E = indexList.end(); It != E;) {
+ IndexListEntry &Entry = *It++;
+ if (Entry.getInstr() || Keep.contains(&Entry)) {
+ Entry.setIndex(Index);
+ Index += SlotIndex::InstrDist;
+ continue;
+ }
+ // Take the surviving entry behind, so a stale index stays in program order.
+ assert(Index != 0 && "First entry is always kept");
+ Entry.setIndex(Index - SlotIndex::InstrDist);
+ Erased.push_back(&Entry);
+ indexList.remove(Entry);
+ }
+
+ for (IndexListEntry *Entry : Erased)
+ Entry->setErased();
+
+ NumDeadEntriesErased += Erased.size();
+ return Erased.size();
+}
+
void SlotIndexes::print(raw_ostream &OS) const {
for (const IndexListEntry &ILE : indexList) {
OS << ILE.getIndex() << ' ';
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-compact-slot-indexes-priority.mir b/llvm/test/CodeGen/AMDGPU/greedy-compact-slot-indexes-priority.mir
new file mode 100644
index 00000000000000..a0a9dff1c93637
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/greedy-compact-slot-indexes-priority.mir
@@ -0,0 +1,82 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -start-before=greedy,0 \
+# RUN: -stop-after=virt-reg-rewriter,2 -verify-machineinstrs -verify-regalloc \
+# RUN: -greedy-compact-slot-indexes=false %s -o - | FileCheck %s --check-prefix=PACK
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -start-before=greedy,0 \
+# RUN: -stop-after=virt-reg-rewriter,2 -verify-machineinstrs -verify-regalloc \
+# RUN: -greedy-compact-slot-indexes %s -o - | FileCheck %s --check-prefix=COMPACT
+
+# This checks the impact of removing dead slot indexes.
+# Run the real SGPR, WWM and VGPR allocation pipeline. SGPR rewriting deletes
+# the six identity copies between the two VGPR definitions. With those entries
+# retained, %0 spans more slots than %1 and is allocated first. Removing them
+# makes %1 the longer global range, so it gets the first available VGPR instead.
+
+---
+name: compact_priority
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ ; PACK-LABEL: name: compact_priority
+ ; PACK: bb.0:
+ ; PACK-NEXT: successors: %bb.1(0x80000000)
+ ; PACK-NEXT: liveins: $sgpr0
+ ; PACK-NEXT: {{ $}}
+ ; PACK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 100, implicit $exec
+ ; PACK-NEXT: S_NOP 0, implicit killed renamable $sgpr0
+ ; PACK-NEXT: renamable $vgpr1 = V_MOV_B32_e32 200, implicit $exec
+ ; PACK-NEXT: S_BRANCH %bb.1
+ ; PACK-NEXT: {{ $}}
+ ; PACK-NEXT: bb.1:
+ ; PACK-NEXT: liveins: $vgpr0, $vgpr1
+ ; PACK-NEXT: {{ $}}
+ ; PACK-NEXT: S_NOP 0, implicit killed renamable $vgpr0
+ ; PACK-NEXT: S_NOP 1
+ ; PACK-NEXT: S_NOP 2
+ ; PACK-NEXT: S_NOP 3
+ ; PACK-NEXT: S_NOP 0, implicit killed renamable $vgpr1
+ ; PACK-NEXT: S_ENDPGM 0
+ ;
+ ; COMPACT-LABEL: name: compact_priority
+ ; COMPACT: bb.0:
+ ; COMPACT-NEXT: successors: %bb.1(0x80000000)
+ ; COMPACT-NEXT: liveins: $sgpr0
+ ; COMPACT-NEXT: {{ $}}
+ ; COMPACT-NEXT: renamable $vgpr1 = V_MOV_B32_e32 100, implicit $exec
+ ; COMPACT-NEXT: S_NOP 0, implicit killed renamable $sgpr0
+ ; COMPACT-NEXT: renamable $vgpr0 = V_MOV_B32_e32 200, implicit $exec
+ ; COMPACT-NEXT: S_BRANCH %bb.1
+ ; COMPACT-NEXT: {{ $}}
+ ; COMPACT-NEXT: bb.1:
+ ; COMPACT-NEXT: liveins: $vgpr0, $vgpr1
+ ; COMPACT-NEXT: {{ $}}
+ ; COMPACT-NEXT: S_NOP 0, implicit killed renamable $vgpr1
+ ; COMPACT-NEXT: S_NOP 1
+ ; COMPACT-NEXT: S_NOP 2
+ ; COMPACT-NEXT: S_NOP 3
+ ; COMPACT-NEXT: S_NOP 0, implicit killed renamable $vgpr0
+ ; COMPACT-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1
+ liveins: $sgpr0
+
+ %0:vgpr_32 = V_MOV_B32_e32 100, implicit $exec
+ %2:sreg_32 = COPY $sgpr0
+ %3:sreg_32 = COPY %2
+ %4:sreg_32 = COPY %3
+ %5:sreg_32 = COPY %4
+ %6:sreg_32 = COPY %5
+ %7:sreg_32 = COPY %6
+ S_NOP 0, implicit %7
+ %1:vgpr_32 = V_MOV_B32_e32 200, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ S_NOP 0, implicit %0
+ S_NOP 1
+ S_NOP 2
+ S_NOP 3
+ S_NOP 0, implicit %1
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-compact-slot-indexes.ll b/llvm/test/CodeGen/AMDGPU/greedy-compact-slot-indexes.ll
new file mode 100644
index 00000000000000..307d29721bca63
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/greedy-compact-slot-indexes.ll
@@ -0,0 +1,35 @@
+; RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -O3 -stress-regalloc=8 \
+; RUN: -greedy-compact-slot-indexes -filetype=null < %s
+
+; -greedy-compact-slot-indexes unlinks the SlotIndexes entries of erased
+; instructions, so LiveDebugVariables must not hold an index to one, or
+; emitting its DBG_VALUEs crashes. AMDGPU runs greedy three times (SGPR, WWM,
+; VGPR) against one LiveDebugVariables, and -stress-regalloc=8 makes some debug
+; values point at instructions erased in between.
+
+define amdgpu_kernel void @sdiv_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x, i32 %y) !dbg !5 {
+ %shl.y = shl i32 4096, %y, !dbg !12
+ #dbg_value(i32 %shl.y, !9, !DIExpression(), !12)
+ %r = sdiv i32 %x, %shl.y, !dbg !13
+ #dbg_value(i32 %r, !11, !DIExpression(), !13)
+ store i32 %r, ptr addrspace(1) %out, align 4, !dbg !14
+ ret void, !dbg !15
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "llvm", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "t.ll", directory: "/")
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "sdiv_i32_pow2_shl_denom", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !{!9, !11}
+!9 = !DILocalVariable(name: "1", scope: !5, file: !1, line: 1, type: !10)
+!10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+!11 = !DILocalVariable(name: "2", scope: !5, file: !1, line: 2, type: !10)
+!12 = !DILocation(line: 1, column: 1, scope: !5)
+!13 = !DILocation(line: 2, column: 1, scope: !5)
+!14 = !DILocation(line: 3, column: 1, scope: !5)
+!15 = !DILocation(line: 4, column: 1, scope: !5)
diff --git a/llvm/unittests/CodeGen/SlotIndexesTest.cpp b/llvm/unittests/CodeGen/SlotIndexesTest.cpp
index 1725fbe2190fe5..6b73194cf1210d 100644
--- a/llvm/unittests/CodeGen/SlotIndexesTest.cpp
+++ b/llvm/unittests/CodeGen/SlotIndexesTest.cpp
@@ -8,7 +8,9 @@
#include "llvm/CodeGen/SlotIndexes.h"
#include "CodeGenTestBase.h"
+#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/Config/Targets.h"
#include "llvm/Support/TargetSelect.h"
#include "gtest/gtest.h"
@@ -204,4 +206,290 @@ body: |
EXPECT_TRUE(SI.isBlockBoundaryIndex(SI.getMBBEndIdx(&MBB)));
}
+/// Deletes \p MI the way a real pass does: its index list entry survives.
+static void deleteInstr(MachineInstr &MI, LiveIntervals &LIS) {
+ for (const MachineOperand &MO : MI.all_defs())
+ if (MO.getReg().isVirtual() && LIS.hasInterval(MO.getReg()))
+ LIS.removeInterval(MO.getReg());
+ LIS.RemoveMachineInstrFromMaps(MI);
+ MI.eraseFromParent();
+}
+
+/// Erases \p MI without retiring the interval of the register it defines,
+/// which is the bug removeUnreferencedIntervals() repairs.
+static void orphanInstr(MachineInstr &MI, LiveIntervals &LIS) {
+ LIS.RemoveMachineInstrFromMaps(MI);
+ MI.eraseFromParent();
+}
+
+/// Compacts the way RAGreedy does.
+static unsigned compact(LiveIntervals &LIS) {
+ LIS.removeUnreferencedIntervals();
+ return LIS.getSlotIndexes()->compactIndexes();
+}
+
+/// Leftover entries inflate live range sizes, which is what misled the
+/// allocator. Compaction must restore the real def-to-use size.
+TEST_F(SlotIndexesTest, CompactIndexesShrinksSizeToRealDistance) {
+ StringRef MIRString = R"(
+---
+name: func
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %3:vgpr_32 = IMPLICIT_DEF
+ S_NOP 0, implicit %0
+ S_ENDPGM 0
+...
+ )";
+ ASSERT_TRUE(parseMIR(MIRString));
+
+ MachineFunction &MF = getMF("func");
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ MachineBasicBlock &MBB = *MF.getBlockNumbered(0);
+
+ const Register Tracked = Register::index2VirtReg(0);
+ EXPECT_EQ(LIS.getInterval(Tracked).getSize(), 4u * SlotIndex::InstrDist);
+
+ for (Register Reg : {Register::index2VirtReg(1), Register::index2VirtReg(2),
+ Register::index2VirtReg(3)})
+ deleteInstr(*MF.getRegInfo().getVRegDef(Reg), LIS);
+ ASSERT_EQ(std::distance(MBB.begin(), MBB.end()), 3L);
+
+ // Deleting does not renumber, so the size is unchanged.
+ EXPECT_EQ(LIS.getInterval(Tracked).getSize(), 4u * SlotIndex::InstrDist);
+
+ EXPECT_EQ(compact(LIS), 3u);
+
+ EXPECT_EQ(LIS.getInterval(Tracked).getSize(), 1u * SlotIndex::InstrDist);
+ EXPECT_TRUE(MF.verify(&LIS, LIS.getSlotIndexes(), /*Banner=*/nullptr,
+ /*OS=*/&errs(), /*AbortOnError=*/false));
+}
+
+/// An interval left behind for a register no instruction mentions holds its
+/// segment start and value number on an erased index. Retiring it is what
+/// lets compaction erase that entry rather than having to keep it alive.
+TEST_F(SlotIndexesTest, RemoveUnreferencedIntervalsRetiresOrphanedInterval) {
+ StringRef MIRString = R"(
+---
+name: func
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ S_NOP 0, implicit %0
+ S_ENDPGM 0
+...
+ )";
+ ASSERT_TRUE(parseMIR(MIRString));
+
+ MachineFunction &MF = getMF("func");
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ SlotIndexes &SI = *LIS.getSlotIndexes();
+
+ const Register Orphan = Register::index2VirtReg(1);
+ orphanInstr(*MF.getRegInfo().getVRegDef(Orphan), LIS);
+
+ // The interval outlives its only instruction, on a now-erased index.
+ ASSERT_TRUE(LIS.hasInterval(Orphan));
+ EXPECT_TRUE(SI.isStaleIndex(LIS.getInterval(Orphan).beginIndex()));
+
+ EXPECT_EQ(LIS.removeUnreferencedIntervals(), 1u);
+ EXPECT_FALSE(LIS.hasInterval(Orphan));
+
+ // With nothing holding it, the entry is erased rather than kept alive.
+ EXPECT_EQ(SI.compactIndexes(), 1u);
+ EXPECT_TRUE(MF.verify(&LIS, &SI, /*Banner=*/nullptr, /*OS=*/&errs(),
+ /*AbortOnError=*/false));
+}
+
+/// A register kept alive only by a debug use must not be retired, or what
+/// gets removed would depend on -g.
+TEST_F(SlotIndexesTest, RemoveUnreferencedIntervalsKeepsDebugOnlyRegister) {
+ StringRef MIRString = R"(
+--- |
+ define amdgpu_kernel void @func() !dbg !5 {
+ ret void
+ }
+
+ !llvm.dbg.cu = !{!0}
+ !llvm.module.flags = !{!4}
+
+ !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "llvm", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+ !1 = !DIFile(filename: "t.ll", directory: "/")
+ !4 = !{i32 2, !"Debug Info Version", i32 3}
+ !5 = distinct !DISubprogram(name: "func", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+ !6 = !DISubroutineType(types: !7)
+ !7 = !{}
+ !8 = !{!9}
+ !9 = !DILocalVariable(name: "v", scope: !5, file: !1, line: 1, type: !10)
+ !10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+ !12 = !DILocation(line: 1, column: 1, scope: !5)
+...
+---
+name: func
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ DBG_VALUE %1, $noreg, !9, !DIExpression(), debug-location !12
+ S_NOP 0, implicit %0
+ S_ENDPGM 0
+...
+ )";
+ ASSERT_TRUE(parseMIR(MIRString));
+
+ MachineFunction &MF = getMF("func");
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ MachineRegisterInfo &MRI = MF.getRegInfo();
+
+ // Erasing the def leaves the DBG_VALUE as the only remaining reference.
+ const Register Orphan = Register::index2VirtReg(1);
+ orphanInstr(*MRI.getVRegDef(Orphan), LIS);
+
+ ASSERT_FALSE(MRI.reg_empty(Orphan));
+ ASSERT_TRUE(MRI.reg_nodbg_empty(Orphan));
+ EXPECT_EQ(LIS.removeUnreferencedIntervals(), 0u);
+ EXPECT_TRUE(LIS.hasInterval(Orphan));
+}
+
+/// Block boundary entries carry no instruction by design, so compaction must
+/// not mistake them for leftovers.
+TEST_F(SlotIndexesTest, CompactIndexesKeepsBlockBoundaries) {
+ StringRef MIRString = R"(
+---
+name: func
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ S_CMP_EQ_U32 0, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ %2:vgpr_32 = IMPLICIT_DEF
+ S_NOP 0, implicit %0
+
+ bb.2:
+ S_ENDPGM 0
+...
+ )";
+ ASSERT_TRUE(parseMIR(MIRString));
+
+ MachineFunction &MF = getMF("func");
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ SlotIndexes &SI = *LIS.getSlotIndexes();
+
+ deleteInstr(*MF.getRegInfo().getVRegDef(Register::index2VirtReg(1)), LIS);
+ deleteInstr(*MF.getRegInfo().getVRegDef(Register::index2VirtReg(2)), LIS);
+
+ EXPECT_EQ(compact(LIS), 2u);
+
+ for (MachineBasicBlock &MBB : MF) {
+ SlotIndex Start = LIS.getMBBStartIdx(&MBB);
+ EXPECT_TRUE(Start.isValid());
+ EXPECT_EQ(SI.getMBBFromIndex(Start), &MBB);
+ EXPECT_LT(Start, LIS.getMBBEndIdx(&MBB));
+ }
+
+ EXPECT_TRUE(MF.verify(&LIS, &SI, /*Banner=*/nullptr, /*OS=*/&errs(),
+ /*AbortOnError=*/false));
+}
+
+/// An unreported index still has to compare sanely, since only an asserts
+/// build diagnoses it. It takes the index of the surviving entry behind it.
+TEST_F(SlotIndexesTest, CompactIndexesKeepsStaleIndexesInProgramOrder) {
+ StringRef MIRString = R"(
+---
+name: func
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %3:vgpr_32 = IMPLICIT_DEF
+ S_NOP 0, implicit %0
+ S_ENDPGM 0
+...
+ )";
+ ASSERT_TRUE(parseMIR(MIRString));
+
+ MachineFunction &MF = getMF("func");
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ SlotIndexes &SI = *LIS.getSlotIndexes();
+ MachineBasicBlock &MBB = *MF.getBlockNumbered(0);
+
+ const SlotIndex Behind = LIS.getInstructionIndex(*MBB.begin()).getBaseIndex();
+ SmallVector<SlotIndex, 3> Stale;
+ for (unsigned I : {1u, 2u, 3u}) {
+ MachineInstr &MI = *MF.getRegInfo().getVRegDef(Register::index2VirtReg(I));
+ Stale.push_back(LIS.getInstructionIndex(MI).getBaseIndex());
+ }
+ for (unsigned I : {1u, 2u, 3u})
+ deleteInstr(*MF.getRegInfo().getVRegDef(Register::index2VirtReg(I)), LIS);
+
+ ASSERT_EQ(compact(LIS), 3u);
+
+ for (SlotIndex S : Stale) {
+ EXPECT_LT(S, SI.getLastIndex());
+ // Ordered with the entry behind, but no longer distinct from it.
+ EXPECT_FALSE(S < Behind);
+ EXPECT_FALSE(Behind < S);
+ EXPECT_FALSE(S == Behind);
+ }
+}
+
+/// Holding an index to an erased instruction across compaction is a bug in
+/// the holder, so it is diagnosed at the point of use rather than followed.
+#if !defined(NDEBUG) && defined(GTEST_HAS_DEATH_TEST)
+TEST_F(SlotIndexesTest, CompactIndexesFlagsStaleIndexUse) {
+ StringRef MIRString = R"(
+---
+name: func
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ S_NOP 0, implicit %0
+ S_ENDPGM 0
+...
+ )";
+ ASSERT_TRUE(parseMIR(MIRString));
+
+ MachineFunction &MF = getMF("func");
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ SlotIndexes &SI = *LIS.getSlotIndexes();
+
+ MachineInstr &Dead = *MF.getRegInfo().getVRegDef(Register::index2VirtReg(1));
+ // An outside holder that kept this index past the instruction.
+ const SlotIndex Stale = LIS.getInstructionIndex(Dead).getBaseIndex();
+ deleteInstr(Dead, LIS);
+
+ ASSERT_EQ(compact(LIS), 1u);
+
+ EXPECT_DEATH((void)SI.getInstructionFromIndex(Stale),
+ "SlotIndex outlived the instruction it pointed at");
+}
+#endif
+
} // namespace
>From ea1b5425bc714b6fe03e8f7bb7eb4f3e5537c713 Mon Sep 17 00:00:00 2001
From: Andreas Jonson <andjo403 at hotmail.com>
Date: Sat, 24 Oct 2026 14:32:34 +0200
Subject: [PATCH 24/37] [LVI] constant fold not condition if val has condition
as operand. (#235278)
proof: https://alive2.llvm.org/ce/z/fq0q2N
---
llvm/lib/Analysis/LazyValueInfo.cpp | 9 +++++---
.../CorrelatedValuePropagation/basic.ll | 21 +++++++++++++++++++
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index b15cee2fd6e9fa..b28f0bd4d9b456 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -1662,7 +1662,10 @@ LazyValueInfoImpl::getEdgeValueLocal(Value *Val, BasicBlock *BBFrom,
// instructions with many operands.
if (isa<IntegerType>(Usr->getType()) && isOperationFoldable(Usr)) {
const DataLayout &DL = BBTo->getDataLayout();
- if (usesOperand(Usr, Condition)) {
+ Value *TempCond = Condition;
+ if (usesOperand(Usr, Condition) ||
+ (match(Condition, m_Not(m_Value(TempCond))) &&
+ usesOperand(Usr, TempCond))) {
// If Val has Condition as an operand and Val can be folded into a
// constant with either Condition == true or Condition == false,
// propagate the constant.
@@ -1670,8 +1673,8 @@ LazyValueInfoImpl::getEdgeValueLocal(Value *Val, BasicBlock *BBFrom,
// ; %Val is true on the edge to %then.
// %Val = and i1 %Condition, true.
// br %Condition, label %then, label %else
- APInt ConditionVal(1, isTrueDest ? 1 : 0);
- Result = constantFoldUser(Usr, Condition, ConditionVal, DL);
+ APInt ConditionVal(1, isTrueDest ^ (TempCond != Condition) ? 1 : 0);
+ Result = constantFoldUser(Usr, TempCond, ConditionVal, DL);
} else if (isa<TruncInst, ZExtInst, SExtInst>(Usr)) {
ValueLatticeElement OpLatticeVal =
*getValueFromCondition(Usr->getOperand(0), Condition,
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
index a0175f3ebdd731..c44bab391b3e32 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
@@ -2091,6 +2091,27 @@ define range(i32 512, 1024) i32 @range_non_overlapping(i8 %x) {
ret i32 %zext
}
+define i32 @sext_cond_br_not_cond(i1 %a) {
+; CHECK-LABEL: define range(i32 -1, 1) i32 @sext_cond_br_not_cond
+; CHECK-SAME: (i1 [[A:%.*]]) {
+; CHECK-NEXT: [[B:%.*]] = sext i1 [[A]] to i32
+; CHECK-NEXT: [[NOT:%.*]] = xor i1 [[A]], true
+; CHECK-NEXT: br i1 [[NOT]], label [[FALSE:%.*]], label [[TRUE:%.*]]
+; CHECK: true:
+; CHECK-NEXT: ret i32 -1
+; CHECK: false:
+; CHECK-NEXT: ret i32 0
+;
+ %b = sext i1 %a to i32
+ %not = xor i1 %a , true
+ br i1 %not, label %false, label %true
+
+true:
+ ret i32 %b
+false:
+ ret i32 %b
+}
+
declare i32 @llvm.uadd.sat.i32(i32, i32)
declare i32 @llvm.usub.sat.i32(i32, i32)
declare i32 @llvm.sadd.sat.i32(i32, i32)
>From 99e46680a916254df905aaa1292ed5b48916bf3b Mon Sep 17 00:00:00 2001
From: Hank Chang <hank.chang at sifive.com>
Date: Sat, 24 Oct 2026 21:34:05 +0800
Subject: [PATCH 25/37] [RISCV][P-ext] Add isel pattern match for
mulu.h01/mulu.h11 (#235226)
`mulu.h01` multiplies the zero-extended lower half word of rs1 by the
zero-extended upper half word of rs2, and `mulu.h11` multiplies the
zero-extended upper half words of both.
Previously, (srl X, 16) was selected as a separate srli feeding
`mulu.h00` or `mul`. Match it directly so the shift is folded.
zexti16 also matches (srl X, 16) since its upper bits are known zero, so
the new patterns need AddedComplexity to be tried before the `mulu.h00`
pattern, and `mulu.h11` before `mulu.h01`.
---
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 10 ++++
llvm/test/CodeGen/RISCV/rvp-scalar.ll | 71 ++++++++++++++++++++++++
2 files changed, 81 insertions(+)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index b962fe58241a9c..4b192a236d4148 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -2672,6 +2672,16 @@ let append Predicates = [IsRV32] in {
def : Pat<(i32 (mul (sra GPR:$rs1, (XLenVT 16)), (sra GPR:$rs2, (XLenVT 16)))),
(MUL_H11 GPR:$rs1, GPR:$rs2)>;
+ // zexti16 also matches (srl X, 16), so these need extra complexity to be
+ // preferred over the MULU_H00 pattern above and to have MULU_H11 preferred
+ // over MULU_H01.
+ let AddedComplexity = 1 in
+ def : Pat<(i32 (mul zexti16:$rs1, (srl GPR:$rs2, (XLenVT 16)))),
+ (MULU_H01 zexti16:$rs1, GPR:$rs2)>;
+ let AddedComplexity = 2 in
+ def : Pat<(i32 (mul (srl GPR:$rs1, (XLenVT 16)), (srl GPR:$rs2, (XLenVT 16)))),
+ (MULU_H11 GPR:$rs1, GPR:$rs2)>;
+
def : Pat<(i32 (add GPR:$rd, (mul_oneuse sexti16:$rs1,
(sra GPR:$rs2, (XLenVT 16))))),
(MACC_H01 GPR:$rd, sexti16:$rs1, GPR:$rs2)>;
diff --git a/llvm/test/CodeGen/RISCV/rvp-scalar.ll b/llvm/test/CodeGen/RISCV/rvp-scalar.ll
index 0aa07a99dde1d7..dcf448a3ad97a9 100644
--- a/llvm/test/CodeGen/RISCV/rvp-scalar.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-scalar.ll
@@ -3321,6 +3321,77 @@ define i32 @mulu_h00_zexti16_and_commute(i16 zeroext %a, i32 %b) nounwind {
ret i32 %mul
}
+define i32 @mulu_h01(i16 zeroext %a, i32 %b) nounwind {
+; RV32-LABEL: mulu_h01:
+; RV32: # %bb.0:
+; RV32-NEXT: mulu.h01 a0, a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: mulu_h01:
+; RV64: # %bb.0:
+; RV64-NEXT: srliw a1, a1, 16
+; RV64-NEXT: mulw a0, a0, a1
+; RV64-NEXT: ret
+ %aext = zext i16 %a to i32
+ %bhi = lshr i32 %b, 16
+ %mul = mul i32 %aext, %bhi
+ ret i32 %mul
+}
+
+define i32 @mulu_h01_commute(i16 zeroext %a, i32 %b) nounwind {
+; RV32-LABEL: mulu_h01_commute:
+; RV32: # %bb.0:
+; RV32-NEXT: mulu.h01 a0, a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: mulu_h01_commute:
+; RV64: # %bb.0:
+; RV64-NEXT: srliw a1, a1, 16
+; RV64-NEXT: mulw a0, a1, a0
+; RV64-NEXT: ret
+ %aext = zext i16 %a to i32
+ %bhi = lshr i32 %b, 16
+ %mul = mul i32 %bhi, %aext
+ ret i32 %mul
+}
+
+; Low halfword comes from an and rather than a zeroext argument.
+define i32 @mulu_h01_and(i32 %a, i32 %b) nounwind {
+; RV32-LABEL: mulu_h01_and:
+; RV32: # %bb.0:
+; RV32-NEXT: mulu.h01 a0, a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: mulu_h01_and:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.h a0, a0
+; RV64-NEXT: srliw a1, a1, 16
+; RV64-NEXT: mulw a0, a0, a1
+; RV64-NEXT: ret
+ %alo = and i32 %a, 65535
+ %bhi = lshr i32 %b, 16
+ %mul = mul i32 %alo, %bhi
+ ret i32 %mul
+}
+
+define i32 @mulu_h11(i32 %a, i32 %b) nounwind {
+; RV32-LABEL: mulu_h11:
+; RV32: # %bb.0:
+; RV32-NEXT: mulu.h11 a0, a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: mulu_h11:
+; RV64: # %bb.0:
+; RV64-NEXT: srliw a0, a0, 16
+; RV64-NEXT: srliw a1, a1, 16
+; RV64-NEXT: mulw a0, a0, a1
+; RV64-NEXT: ret
+ %ahi = lshr i32 %a, 16
+ %bhi = lshr i32 %b, 16
+ %mul = mul i32 %ahi, %bhi
+ ret i32 %mul
+}
+
define i32 @mulsu_h00_sexti16_and(i16 signext %a, i32 %b) nounwind {
; RV32-LABEL: mulsu_h00_sexti16_and:
; RV32: # %bb.0:
>From 44c618a4425eb8b83bf2f35df5e9d573c12a65d0 Mon Sep 17 00:00:00 2001
From: Koakuma <koachan at protonmail.com>
Date: Sat, 24 Oct 2026 20:49:29 +0700
Subject: [PATCH 26/37] [SPARC] Implement
TargetInstrInfo::insertIndirectBranch() (#226678)
Implement `insertIndirectBranch` so that branch relaxation pass can
insert unconditional long-range branching.
Follow an approach similar to the one used in AVR and use the CALL
instruction to get a reach of 32-bit signed offset instead of emitting
actual indirect branches.
---
llvm/lib/Target/Sparc/SparcFrameLowering.cpp | 50 ++-
llvm/lib/Target/Sparc/SparcFrameLowering.h | 5 +
llvm/lib/Target/Sparc/SparcInstrInfo.cpp | 75 +++-
llvm/lib/Target/Sparc/SparcInstrInfo.h | 5 +
llvm/lib/Target/Sparc/SparcInstrInfo.td | 11 +
.../Target/Sparc/SparcMachineFunctionInfo.h | 19 +
llvm/lib/Target/Sparc/SparcRegisterInfo.h | 4 +
llvm/test/CodeGen/SPARC/branches-relax.ll | 368 +++++++++++++++++-
8 files changed, 532 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/Sparc/SparcFrameLowering.cpp b/llvm/lib/Target/Sparc/SparcFrameLowering.cpp
index 53e6523aee4e16..9abaef7b32f51c 100644
--- a/llvm/lib/Target/Sparc/SparcFrameLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcFrameLowering.cpp
@@ -20,14 +20,18 @@
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
using namespace llvm;
+#define DEBUG_TYPE "sparc-frame-lowering"
+
SparcFrameLowering::SparcFrameLowering(const SparcSubtarget &ST)
: TargetFrameLowering(TargetFrameLowering::StackGrowsDown,
ST.is64Bit() ? Align(16) : Align(8), 0,
ST.is64Bit() ? Align(16) : Align(8),
- /*StackRealignable=*/false) {}
+ /*StackRealignable=*/false),
+ STI(ST) {}
void SparcFrameLowering::emitSPAdjustment(MachineFunction &MF,
MachineBasicBlock &MBB,
@@ -256,12 +260,14 @@ bool SparcFrameLowering::isLeafProc(MachineFunction &MF) const
MachineRegisterInfo &MRI = MF.getRegInfo();
MachineFrameInfo &MFI = MF.getFrameInfo();
+ SparcMachineFunctionInfo *SMFI = MF.getInfo<SparcMachineFunctionInfo>();
return !(MFI.hasCalls() // has calls
|| MRI.isPhysRegUsed(SP::L0) // Too many registers needed
|| MRI.isPhysRegUsed(SP::O6) // %sp is used
|| hasFP(MF) // need %fp
- || MF.hasInlineAsm()); // has inline assembly
+ || MF.hasInlineAsm() // has inline assembly
+ || SMFI->isLargeProc()); // is a large procedure
}
void SparcFrameLowering::remapRegsForLeafProc(MachineFunction &MF) const {
@@ -306,15 +312,53 @@ void SparcFrameLowering::remapRegsForLeafProc(MachineFunction &MF) const {
#endif
}
+static uint64_t estimateFunctionSizeInBytes(const SparcInstrInfo *TII,
+ const MachineFunction &MF) {
+ uint64_t FuncSize = 0;
+ for (auto &MBB : MF)
+ for (auto &MI : MBB) {
+ // Be conservative here and count the DS too.
+ uint64_t Multiplier = MI.hasDelaySlot() ? 2 : 1;
+ FuncSize += (TII->getInstSizeInBytes(MI) * Multiplier);
+ }
+ return FuncSize;
+}
+
void SparcFrameLowering::determineCalleeSaves(MachineFunction &MF,
BitVector &SavedRegs,
RegScavenger *RS) const {
TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+ SparcMachineFunctionInfo *MFI = MF.getInfo<SparcMachineFunctionInfo>();
+ const SparcInstrInfo *TII = STI.getInstrInfo();
const SparcSubtarget &Subtarget = MF.getSubtarget<SparcSubtarget>();
+
+ bool IsLargeFunction = !isInt<23>(estimateFunctionSizeInBytes(TII, MF));
+ MFI->setLargeProc(IsLargeFunction);
+
if (!Subtarget.getCLOpts().disable_sparc_leaf_proc && isLeafProc(MF)) {
- SparcMachineFunctionInfo *MFI = MF.getInfo<SparcMachineFunctionInfo>();
MFI->setLeafProc(true);
remapRegsForLeafProc(MF);
}
}
+
+void SparcFrameLowering::processFunctionBeforeFrameFinalized(
+ MachineFunction &MF, RegScavenger *RS) const {
+ const SparcRegisterInfo *RI = STI.getRegisterInfo();
+ const TargetRegisterClass &RC =
+ STI.is64Bit() ? SP::I64RegsRegClass : SP::IntRegsRegClass;
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ SparcMachineFunctionInfo *SMFI = MF.getInfo<SparcMachineFunctionInfo>();
+
+ // Far branches beyond 23-bit offset require a spill slot for scratch
+ // register.
+ if (SMFI->isLargeProc()) {
+ int FI =
+ MFI.CreateSpillStackObject(RI->getSpillSize(RC), RI->getSpillAlign(RC));
+ RS->addScavengingFrameIndex(FI);
+ if (SMFI->getBranchRelaxationSpillFrameIndex() == -1)
+ SMFI->setBranchRelaxationSpillFrameIndex(FI);
+ LLVM_DEBUG(dbgs() << "Allocated FI(" << FI
+ << ") as the emergency spill slot.\n");
+ }
+}
diff --git a/llvm/lib/Target/Sparc/SparcFrameLowering.h b/llvm/lib/Target/Sparc/SparcFrameLowering.h
index 803856811969b2..0fb3c0589bc269 100644
--- a/llvm/lib/Target/Sparc/SparcFrameLowering.h
+++ b/llvm/lib/Target/Sparc/SparcFrameLowering.h
@@ -21,6 +21,8 @@ namespace llvm {
class SparcSubtarget;
class SparcFrameLowering : public TargetFrameLowering {
+ const SparcSubtarget &STI;
+
public:
explicit SparcFrameLowering(const SparcSubtarget &ST);
@@ -46,6 +48,9 @@ class SparcFrameLowering : public TargetFrameLowering {
/// time).
bool targetHandlesStackFrameRounding() const override { return true; }
+ void processFunctionBeforeFrameFinalized(MachineFunction &MF,
+ RegScavenger *RS) const override;
+
protected:
bool hasFPImpl(const MachineFunction &MF) const override;
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
index 9f846b7a49373c..d3bb79a7769e4c 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -19,6 +19,7 @@
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/Support/ErrorHandling.h"
using namespace llvm;
@@ -264,7 +265,8 @@ SparcInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
case SP::BPRA:
case SP::BPRNT:
case SP::BPRANT:
- return MI.getOperand(0).getMBB();
+ case SP::CALL_LONGJUMP:
+ return MI.getOperand(0).getMBB();
}
}
@@ -391,6 +393,74 @@ unsigned SparcInstrInfo::insertBranch(MachineBasicBlock &MBB,
return 2;
}
+void SparcInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
+ MachineBasicBlock &NewDestBB,
+ MachineBasicBlock &RestoreBB,
+ const DebugLoc &DL, int64_t BrOffset,
+ RegScavenger *RS) const {
+ assert(RS && "RegScavenger required for long branching");
+ assert(MBB.empty() &&
+ "new block should be inserted for expanding unconditional branch");
+ assert(MBB.pred_size() == 1 && "should only have a single predecessor");
+ assert(RestoreBB.empty() &&
+ "restore block should be inserted for restoring clobbered registers");
+
+ MachineFunction *MF = MBB.getParent();
+ SparcMachineFunctionInfo *SMFI = MF->getInfo<SparcMachineFunctionInfo>();
+
+ if (!isInt<32>(BrOffset))
+ report_fatal_error(
+ "Branch offsets outside of the signed 32-bit range not supported");
+ RS->enterBasicBlockEnd(MBB);
+
+ // A SPARC CALL instruction already has signed 32-bit range, so we'll use it
+ // instead of actually emitting an indirect branch.
+ //
+ // Note that using CALLs to do unconditional long-range branch in this manner
+ // should only be done if the target is out of range of a regular BA
+ // instruction, given that CALLS will always clobber %o7 and interfere with
+ // the processor's predictor machinery.
+ bool IsO7Used = RS->isRegUsed(SP::O7);
+ Register ScratchReg = RS->FindUnusedReg(&SP::IntRegsRegClass);
+ int FI = SMFI->getBranchRelaxationSpillFrameIndex();
+ if (IsO7Used) {
+ if (ScratchReg.isValid()) {
+ BuildMI(&MBB, DL, get(SP::ORrr), ScratchReg)
+ .addReg(SP::G0)
+ .addReg(SP::O7, RegState::Kill);
+ RS->setRegUsed(ScratchReg);
+ } else {
+ // In case we run out of usable registers we'll spill it into the stack.
+ if (FI == -1)
+ report_fatal_error("The function size is incorrectly estimated.");
+ const MCRegisterClass &RC =
+ Subtarget.is64Bit() ? SP::I64RegsRegClass : SP::IntRegsRegClass;
+
+ MachineBasicBlock::iterator I = MBB.end();
+ storeRegToStackSlot(MBB, I, SP::O7, true, FI, &RC, Register());
+ TRI.eliminateFrameIndex(std::prev(I), 0, 0, RS);
+ }
+ }
+
+ BuildMI(&MBB, DL, get(SP::CALL_LONGJUMP))
+ .addMBB(IsO7Used ? &RestoreBB : &NewDestBB);
+
+ if (IsO7Used) {
+ if (ScratchReg.isValid()) {
+ BuildMI(&RestoreBB, DL, get(SP::ORrr), SP::O7)
+ .addReg(SP::G0)
+ .addReg(ScratchReg, RegState::Kill);
+ } else {
+ const MCRegisterClass &RC =
+ Subtarget.is64Bit() ? SP::I64RegsRegClass : SP::IntRegsRegClass;
+
+ MachineBasicBlock::iterator I = RestoreBB.begin();
+ loadRegFromStackSlot(RestoreBB, I, SP::O7, FI, &RC, Register());
+ TRI.eliminateFrameIndex(std::prev(I), 0, 1, RS);
+ }
+ }
+}
+
unsigned SparcInstrInfo::removeBranch(MachineBasicBlock &MBB,
int *BytesRemoved) const {
MachineBasicBlock::iterator I = MBB.end();
@@ -429,6 +499,9 @@ bool SparcInstrInfo::isBranchOffsetInRange(unsigned BranchOpc,
int64_t Offset) const {
assert((Offset & 0b11) == 0 && "Malformed branch offset");
switch (BranchOpc) {
+ case SP::CALL_LONGJUMP:
+ return isIntN(30, Offset >> 2);
+
case SP::BA:
case SP::BCOND:
case SP::BCONDA:
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.h b/llvm/lib/Target/Sparc/SparcInstrInfo.h
index a719391559ebaf..a6b32df23062f6 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.h
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.h
@@ -82,6 +82,11 @@ class SparcInstrInfo : public SparcGenInstrInfo {
const DebugLoc &DL,
int *BytesAdded = nullptr) const override;
+ void insertIndirectBranch(MachineBasicBlock &MBB,
+ MachineBasicBlock &NewDestBB,
+ MachineBasicBlock &RestoreBB, const DebugLoc &DL,
+ int64_t BrOffset, RegScavenger *RS) const override;
+
bool
reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.td b/llvm/lib/Target/Sparc/SparcInstrInfo.td
index db811db78988ad..b5558fe421994e 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.td
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.td
@@ -1022,6 +1022,17 @@ multiclass IPredBranch<string regstr, list<dag> CCPattern> {
} // let isBranch = 1, isTerminator = 1, hasDelaySlot = 1
+// This is for using the CALL instruction as a longer unconditional branch.
+let Defs = [O7], isBranch = 1, isCall = 0, isCodeGenOnly = 1,
+ isTerminator = 1, hasDelaySlot = 1, isBarrier = 1 in {
+ def CALL_LONGJUMP : InstSP<(outs), (ins calltarget:$disp, variable_ops),
+ "call $disp", []> {
+ bits<30> disp;
+ let op = 1;
+ let Inst{29-0} = disp;
+ }
+}
+
// Indirect branch instructions.
let isTerminator = 1, isBarrier = 1, hasDelaySlot = 1, isBranch =1,
isIndirectBranch = 1, rd = 0, isCodeGenOnly = 1 in {
diff --git a/llvm/lib/Target/Sparc/SparcMachineFunctionInfo.h b/llvm/lib/Target/Sparc/SparcMachineFunctionInfo.h
index f73cd6d1b13171..4ed1a2b2a08cf2 100644
--- a/llvm/lib/Target/Sparc/SparcMachineFunctionInfo.h
+++ b/llvm/lib/Target/Sparc/SparcMachineFunctionInfo.h
@@ -30,6 +30,15 @@ namespace llvm {
/// IsLeafProc - True if the function is a leaf procedure.
bool IsLeafProc;
+
+ /// IsLeafProc - True if the function is larger than the coverage of `ba`
+ /// instruction.
+ bool IsLargeProc = false;
+
+ /// BranchRelaxationSpillFrameIndex - FrameIndex of the spill slot when
+ /// there is no scavenged register in insertIndirectBranch.
+ int BranchRelaxationSpillFrameIndex = -1;
+
public:
SparcMachineFunctionInfo()
: GlobalBaseReg(0), VarArgsFrameOffset(0), SRetReturnReg(0),
@@ -54,6 +63,16 @@ namespace llvm {
void setLeafProc(bool rhs) { IsLeafProc = rhs; }
bool isLeafProc() const { return IsLeafProc; }
+
+ void setLargeProc(bool rhs) { IsLargeProc = rhs; }
+ bool isLargeProc() const { return IsLargeProc; }
+
+ int getBranchRelaxationSpillFrameIndex() {
+ return BranchRelaxationSpillFrameIndex;
+ }
+ void setBranchRelaxationSpillFrameIndex(int Index) {
+ BranchRelaxationSpillFrameIndex = Index;
+ }
};
}
diff --git a/llvm/lib/Target/Sparc/SparcRegisterInfo.h b/llvm/lib/Target/Sparc/SparcRegisterInfo.h
index 30f0b9f191383d..12e650d4227b49 100644
--- a/llvm/lib/Target/Sparc/SparcRegisterInfo.h
+++ b/llvm/lib/Target/Sparc/SparcRegisterInfo.h
@@ -43,6 +43,10 @@ struct SparcRegisterInfo : public SparcGenRegisterInfo {
RegScavenger *RS = nullptr) const override;
Register getFrameRegister(const MachineFunction &MF) const override;
+
+ bool requiresRegisterScavenging(const MachineFunction &MF) const override {
+ return true;
+ }
};
} // end namespace llvm
diff --git a/llvm/test/CodeGen/SPARC/branches-relax.ll b/llvm/test/CodeGen/SPARC/branches-relax.ll
index 334a7ebfb67394..5dcb78237df6fc 100644
--- a/llvm/test/CodeGen/SPARC/branches-relax.ll
+++ b/llvm/test/CodeGen/SPARC/branches-relax.ll
@@ -1,7 +1,49 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=sparc64 -sparc-bpcc-offset-bits=4 -sparc-bpr-offset-bits=4 | FileCheck --check-prefix=SPARC64 %s
+; RUN: llc < %s -mtriple=sparc -verify-machineinstrs | FileCheck --check-prefix=SPARC %s
+; RUN: llc < %s -mtriple=sparc64 -verify-machineinstrs -sparc-bpcc-offset-bits=4 -sparc-bpr-offset-bits=4 | FileCheck --check-prefix=SPARC64 %s
+
+;; Note that 32-bit SPARC branches all have +- 8 MiB range (i.e. same with `ba`)
+;; so for it only @branch_relax_large* test is really relevant.
define i32 @branch_relax_int(i32 %in) {
+; SPARC-LABEL: branch_relax_int:
+; SPARC: .cfi_startproc
+; SPARC-NEXT: ! %bb.0:
+; SPARC-NEXT: save %sp, -96, %sp
+; SPARC-NEXT: .cfi_def_cfa_register %fp
+; SPARC-NEXT: .cfi_window_save
+; SPARC-NEXT: .cfi_register %o7, %i7
+; SPARC-NEXT: cmp %i0, 0
+; SPARC-NEXT: be .LBB0_2
+; SPARC-NEXT: nop
+; SPARC-NEXT: ! %bb.1: ! %false
+; SPARC-NEXT: !APP
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore %g0, %g0, %o0
+; SPARC-NEXT: .LBB0_2: ! %true
+; SPARC-NEXT: mov 4, %i0
+; SPARC-NEXT: !APP
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore
+;
; SPARC64-LABEL: branch_relax_int:
; SPARC64: .cfi_startproc
; SPARC64-NEXT: ! %bb.0:
@@ -54,6 +96,45 @@ false:
}
define i64 @branch_relax_reg(i64 %in) {
+; SPARC-LABEL: branch_relax_reg:
+; SPARC: .cfi_startproc
+; SPARC-NEXT: ! %bb.0:
+; SPARC-NEXT: save %sp, -96, %sp
+; SPARC-NEXT: .cfi_def_cfa_register %fp
+; SPARC-NEXT: .cfi_window_save
+; SPARC-NEXT: .cfi_register %o7, %i7
+; SPARC-NEXT: orcc %i1, %i0, %g0
+; SPARC-NEXT: be .LBB1_2
+; SPARC-NEXT: nop
+; SPARC-NEXT: ! %bb.1: ! %false
+; SPARC-NEXT: !APP
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: mov %g0, %i0
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore %g0, %g0, %o1
+; SPARC-NEXT: .LBB1_2: ! %true
+; SPARC-NEXT: mov 4, %i1
+; SPARC-NEXT: !APP
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore %g0, %g0, %o0
+;
; SPARC64-LABEL: branch_relax_reg:
; SPARC64: .cfi_startproc
; SPARC64-NEXT: ! %bb.0:
@@ -105,6 +186,50 @@ false:
}
define float @branch_relax_float(float %in) {
+; SPARC-LABEL: branch_relax_float:
+; SPARC: .cfi_startproc
+; SPARC-NEXT: ! %bb.0:
+; SPARC-NEXT: save %sp, -96, %sp
+; SPARC-NEXT: .cfi_def_cfa_register %fp
+; SPARC-NEXT: .cfi_window_save
+; SPARC-NEXT: .cfi_register %o7, %i7
+; SPARC-NEXT: st %i0, [%fp+-4]
+; SPARC-NEXT: ld [%fp+-4], %f1
+; SPARC-NEXT: sethi %hi(.LCPI2_0), %i0
+; SPARC-NEXT: ld [%i0+%lo(.LCPI2_0)], %f0
+; SPARC-NEXT: fcmps %f1, %f0
+; SPARC-NEXT: nop
+; SPARC-NEXT: fbne .LBB2_2
+; SPARC-NEXT: nop
+; SPARC-NEXT: ! %bb.1: ! %true
+; SPARC-NEXT: sethi %hi(.LCPI2_1), %i0
+; SPARC-NEXT: ld [%i0+%lo(.LCPI2_1)], %f0
+; SPARC-NEXT: !APP
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore
+; SPARC-NEXT: .LBB2_2: ! %false
+; SPARC-NEXT: !APP
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: nop
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore
+;
; SPARC64-LABEL: branch_relax_float:
; SPARC64: .cfi_startproc
; SPARC64-NEXT: ! %bb.0:
@@ -162,3 +287,244 @@ false:
call void asm sideeffect "nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop", ""()
ret float 0.0
}
+
+define i32 @branch_relax_large(i32 %in) {
+; SPARC-LABEL: branch_relax_large:
+; SPARC: .cfi_startproc
+; SPARC-NEXT: ! %bb.0:
+; SPARC-NEXT: save %sp, -96, %sp
+; SPARC-NEXT: .cfi_def_cfa_register %fp
+; SPARC-NEXT: .cfi_window_save
+; SPARC-NEXT: .cfi_register %o7, %i7
+; SPARC-NEXT: cmp %i0, 0
+; SPARC-NEXT: bne .LBB3_1
+; SPARC-NEXT: nop
+; SPARC-NEXT: ! %bb.3:
+; SPARC-NEXT: call .LBB3_2
+; SPARC-NEXT: nop
+; SPARC-NEXT: .LBB3_1: ! %false
+; SPARC-NEXT: !APP
+; SPARC-NEXT: .skip 268435456
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore %g0, %g0, %o0
+; SPARC-NEXT: .LBB3_2: ! %true
+; SPARC-NEXT: mov 4, %i0
+; SPARC-NEXT: !APP
+; SPARC-NEXT: .skip 268435456
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore
+;
+; SPARC64-LABEL: branch_relax_large:
+; SPARC64: .cfi_startproc
+; SPARC64-NEXT: ! %bb.0:
+; SPARC64-NEXT: save %sp, -144, %sp
+; SPARC64-NEXT: .cfi_def_cfa_register %fp
+; SPARC64-NEXT: .cfi_window_save
+; SPARC64-NEXT: .cfi_register %o7, %i7
+; SPARC64-NEXT: cmp %i0, 0
+; SPARC64-NEXT: bne %icc, .LBB3_1
+; SPARC64-NEXT: nop
+; SPARC64-NEXT: ! %bb.3:
+; SPARC64-NEXT: call .LBB3_2
+; SPARC64-NEXT: nop
+; SPARC64-NEXT: .LBB3_1: ! %false
+; SPARC64-NEXT: !APP
+; SPARC64-NEXT: .skip 268435456
+; SPARC64-NEXT: !NO_APP
+; SPARC64-NEXT: ret
+; SPARC64-NEXT: restore %g0, %g0, %o0
+; SPARC64-NEXT: .LBB3_2: ! %true
+; SPARC64-NEXT: mov 4, %i0
+; SPARC64-NEXT: !APP
+; SPARC64-NEXT: .skip 268435456
+; SPARC64-NEXT: !NO_APP
+; SPARC64-NEXT: ret
+; SPARC64-NEXT: restore
+ %tst = icmp eq i32 %in, 0
+ br i1 %tst, label %true, label %false
+
+true:
+ call void asm sideeffect ".space 268435456", ""()
+ ret i32 4
+
+false:
+ call void asm sideeffect ".space 268435456", ""()
+ ret i32 0
+}
+
+define i32 @branch_relax_large_o7_used(i32 %in) {
+; SPARC-LABEL: branch_relax_large_o7_used:
+; SPARC: .cfi_startproc
+; SPARC-NEXT: ! %bb.0:
+; SPARC-NEXT: save %sp, -96, %sp
+; SPARC-NEXT: .cfi_def_cfa_register %fp
+; SPARC-NEXT: .cfi_window_save
+; SPARC-NEXT: .cfi_register %o7, %i7
+; SPARC-NEXT: cmp %i0, 0
+; SPARC-NEXT: !APP
+; SPARC-NEXT: .Ltmp0:
+; SPARC-NEXT: call .Ltmp0+8
+; SPARC-NEXT: nop
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: bne .LBB4_1
+; SPARC-NEXT: nop
+; SPARC-NEXT: ! %bb.3:
+; SPARC-NEXT: mov %o7, %i0
+; SPARC-NEXT: call .LBB4_4
+; SPARC-NEXT: nop
+; SPARC-NEXT: .LBB4_1: ! %false
+; SPARC-NEXT: !APP
+; SPARC-NEXT: .skip 268435456
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore %g0, %g0, %o0
+; SPARC-NEXT: .LBB4_4: ! %true
+; SPARC-NEXT: mov %i0, %o7
+; SPARC-NEXT: ! %bb.2: ! %true
+; SPARC-NEXT: !APP
+; SPARC-NEXT: .skip 268435456
+; SPARC-NEXT: !NO_APP
+; SPARC-NEXT: ret
+; SPARC-NEXT: restore %g0, %o7, %o0
+;
+; SPARC64-LABEL: branch_relax_large_o7_used:
+; SPARC64: .cfi_startproc
+; SPARC64-NEXT: ! %bb.0:
+; SPARC64-NEXT: save %sp, -144, %sp
+; SPARC64-NEXT: .cfi_def_cfa_register %fp
+; SPARC64-NEXT: .cfi_window_save
+; SPARC64-NEXT: .cfi_register %o7, %i7
+; SPARC64-NEXT: cmp %i0, 0
+; SPARC64-NEXT: !APP
+; SPARC64-NEXT: .Ltmp0:
+; SPARC64-NEXT: call .Ltmp0+8
+; SPARC64-NEXT: nop
+; SPARC64-NEXT: !NO_APP
+; SPARC64-NEXT: bne %icc, .LBB4_1
+; SPARC64-NEXT: nop
+; SPARC64-NEXT: ! %bb.3:
+; SPARC64-NEXT: mov %o7, %i0
+; SPARC64-NEXT: call .LBB4_4
+; SPARC64-NEXT: nop
+; SPARC64-NEXT: .LBB4_1: ! %false
+; SPARC64-NEXT: !APP
+; SPARC64-NEXT: .skip 268435456
+; SPARC64-NEXT: !NO_APP
+; SPARC64-NEXT: ret
+; SPARC64-NEXT: restore %g0, %g0, %o0
+; SPARC64-NEXT: .LBB4_4: ! %true
+; SPARC64-NEXT: mov %i0, %o7
+; SPARC64-NEXT: ! %bb.2: ! %true
+; SPARC64-NEXT: !APP
+; SPARC64-NEXT: .skip 268435456
+; SPARC64-NEXT: !NO_APP
+; SPARC64-NEXT: ret
+; SPARC64-NEXT: restore %g0, %o7, %o0
+ %tst = icmp eq i32 %in, 0
+ %ret = call i32 asm sideeffect "call .+8\0Anop", "={o7}"()
+ br i1 %tst, label %true, label %false
+
+true:
+ call void asm sideeffect ".space 268435456", ""()
+ ret i32 %ret
+
+false:
+ call void asm sideeffect ".space 268435456", ""()
+ ret i32 0
+}
+
+;; For these we reserve everything except %o7 to make sure it is spilled.
+; RUN: llc < %s -mtriple=sparc -mattr=+reserve-g1 -mattr=+reserve-g2 -mattr=+reserve-g3 -mattr=+reserve-g4 -mattr=+reserve-g5 -mattr=+reserve-g6 -mattr=+reserve-g7 -mattr=+reserve-o0 -mattr=+reserve-o1 -mattr=+reserve-o2 -mattr=+reserve-o3 -mattr=+reserve-o4 -mattr=+reserve-o5 -mattr=+reserve-l1 -mattr=+reserve-l2 -mattr=+reserve-l3 -mattr=+reserve-l4 -mattr=+reserve-l5 -mattr=+reserve-l6 -mattr=+reserve-l7 -mattr=+reserve-l0 -mattr=+reserve-i0 -mattr=+reserve-i1 -mattr=+reserve-i2 -mattr=+reserve-i3 -mattr=+reserve-i4 -mattr=+reserve-i5 -verify-machineinstrs | FileCheck --check-prefix=SPARC-SPILL-O7 %s
+; RUN: llc < %s -mtriple=sparc64 -mattr=+reserve-g1 -mattr=+reserve-g2 -mattr=+reserve-g3 -mattr=+reserve-g4 -mattr=+reserve-g5 -mattr=+reserve-g6 -mattr=+reserve-g7 -mattr=+reserve-o0 -mattr=+reserve-o1 -mattr=+reserve-o2 -mattr=+reserve-o3 -mattr=+reserve-o4 -mattr=+reserve-o5 -mattr=+reserve-l1 -mattr=+reserve-l2 -mattr=+reserve-l3 -mattr=+reserve-l4 -mattr=+reserve-l5 -mattr=+reserve-l6 -mattr=+reserve-l7 -mattr=+reserve-l0 -mattr=+reserve-i0 -mattr=+reserve-i1 -mattr=+reserve-i2 -mattr=+reserve-i3 -mattr=+reserve-i4 -mattr=+reserve-i5 -verify-machineinstrs -sparc-bpcc-offset-bits=4 -sparc-bpr-offset-bits=4 | FileCheck --check-prefix=SPARC64-SPILL-O7 %s
+
+define i32 @branch_relax_large_o7_spilled(i32 %in) {
+; SPARC-SPILL-O7-LABEL: branch_relax_large_o7_spilled:
+; SPARC-SPILL-O7: .cfi_startproc
+; SPARC-SPILL-O7-NEXT: ! %bb.0:
+; SPARC-SPILL-O7-NEXT: save %sp, -96, %sp
+; SPARC-SPILL-O7-NEXT: .cfi_def_cfa_register %fp
+; SPARC-SPILL-O7-NEXT: .cfi_window_save
+; SPARC-SPILL-O7-NEXT: .cfi_register %o7, %i7
+; SPARC-SPILL-O7-NEXT: mov %i0, %o7
+; SPARC-SPILL-O7-NEXT: cmp %o7, 0
+; SPARC-SPILL-O7-NEXT: !APP
+; SPARC-SPILL-O7-NEXT: .Ltmp1:
+; SPARC-SPILL-O7-NEXT: call .Ltmp1+8
+; SPARC-SPILL-O7-NEXT: nop
+; SPARC-SPILL-O7-NEXT: !NO_APP
+; SPARC-SPILL-O7-NEXT: !APP
+; SPARC-SPILL-O7-NEXT: mov %g0, %i0
+; SPARC-SPILL-O7-NEXT: !NO_APP
+; SPARC-SPILL-O7-NEXT: bne .LBB5_1
+; SPARC-SPILL-O7-NEXT: nop
+; SPARC-SPILL-O7-NEXT: ! %bb.3:
+; SPARC-SPILL-O7-NEXT: st %o7, [%fp+-4] ! 4-byte Folded Spill
+; SPARC-SPILL-O7-NEXT: call .LBB5_4
+; SPARC-SPILL-O7-NEXT: nop
+; SPARC-SPILL-O7-NEXT: .LBB5_1: ! %false
+; SPARC-SPILL-O7-NEXT: !APP
+; SPARC-SPILL-O7-NEXT: .skip 268435456
+; SPARC-SPILL-O7-NEXT: !NO_APP
+; SPARC-SPILL-O7-NEXT: ret
+; SPARC-SPILL-O7-NEXT: restore
+; SPARC-SPILL-O7-NEXT: .LBB5_4: ! %true
+; SPARC-SPILL-O7-NEXT: ld [%fp+-4], %o7 ! 4-byte Folded Reload
+; SPARC-SPILL-O7-NEXT: ! %bb.2: ! %true
+; SPARC-SPILL-O7-NEXT: !APP
+; SPARC-SPILL-O7-NEXT: .skip 268435456
+; SPARC-SPILL-O7-NEXT: !NO_APP
+; SPARC-SPILL-O7-NEXT: ret
+; SPARC-SPILL-O7-NEXT: restore %g0, %o7, %o0
+;
+; SPARC64-SPILL-O7-LABEL: branch_relax_large_o7_spilled:
+; SPARC64-SPILL-O7: .cfi_startproc
+; SPARC64-SPILL-O7-NEXT: ! %bb.0:
+; SPARC64-SPILL-O7-NEXT: save %sp, -144, %sp
+; SPARC64-SPILL-O7-NEXT: .cfi_def_cfa_register %fp
+; SPARC64-SPILL-O7-NEXT: .cfi_window_save
+; SPARC64-SPILL-O7-NEXT: .cfi_register %o7, %i7
+; SPARC64-SPILL-O7-NEXT: mov %i0, %o7
+; SPARC64-SPILL-O7-NEXT: cmp %o7, 0
+; SPARC64-SPILL-O7-NEXT: !APP
+; SPARC64-SPILL-O7-NEXT: .Ltmp1:
+; SPARC64-SPILL-O7-NEXT: call .Ltmp1+8
+; SPARC64-SPILL-O7-NEXT: nop
+; SPARC64-SPILL-O7-NEXT: !NO_APP
+; SPARC64-SPILL-O7-NEXT: !APP
+; SPARC64-SPILL-O7-NEXT: mov %g0, %i0
+; SPARC64-SPILL-O7-NEXT: !NO_APP
+; SPARC64-SPILL-O7-NEXT: bne %icc, .LBB5_1
+; SPARC64-SPILL-O7-NEXT: nop
+; SPARC64-SPILL-O7-NEXT: ! %bb.3:
+; SPARC64-SPILL-O7-NEXT: stx %o7, [%fp+2039] ! 8-byte Folded Spill
+; SPARC64-SPILL-O7-NEXT: call .LBB5_4
+; SPARC64-SPILL-O7-NEXT: nop
+; SPARC64-SPILL-O7-NEXT: .LBB5_1: ! %false
+; SPARC64-SPILL-O7-NEXT: !APP
+; SPARC64-SPILL-O7-NEXT: .skip 268435456
+; SPARC64-SPILL-O7-NEXT: !NO_APP
+; SPARC64-SPILL-O7-NEXT: ret
+; SPARC64-SPILL-O7-NEXT: restore
+; SPARC64-SPILL-O7-NEXT: .LBB5_4: ! %true
+; SPARC64-SPILL-O7-NEXT: ldx [%fp+2039], %o7 ! 8-byte Folded Reload
+; SPARC64-SPILL-O7-NEXT: ! %bb.2: ! %true
+; SPARC64-SPILL-O7-NEXT: !APP
+; SPARC64-SPILL-O7-NEXT: .skip 268435456
+; SPARC64-SPILL-O7-NEXT: !NO_APP
+; SPARC64-SPILL-O7-NEXT: ret
+; SPARC64-SPILL-O7-NEXT: restore %g0, %o7, %o0
+ %tst = icmp eq i32 %in, 0
+ %rett = call i32 asm sideeffect "call .+8\0Anop", "={o7}"()
+ %retf = call i32 asm sideeffect "clr %i0", "={i0}"()
+ br i1 %tst, label %true, label %false
+
+true:
+ call void asm sideeffect ".space 268435456", ""()
+ ret i32 %rett
+
+false:
+ call void asm sideeffect ".space 268435456", ""()
+ ret i32 %retf
+}
>From 3f3566ce38398eb8c54dffd6a1f5c0b31d45ae83 Mon Sep 17 00:00:00 2001
From: Jinsong Ji <jinsong.ji at intel.com>
Date: Sat, 24 Oct 2026 10:49:57 -0400
Subject: [PATCH 27/37] [Test][clang-repl]Keep value names in clang-repl
pragma-optimize.cpp (#235305)
The test added by aa6e40c073e6 checks for named values
(entry:, %x.addr, %x), which only appear when clang keeps value names.
That is the default in assertion builds, but non-assertion CI builds
discard them, so the CHECK-NEXT: entry: line fails. Pass
-fno-discard-value-names explicitly.
Assisted-By: Claude Opus 5.5
---
clang/test/Interpreter/pragma-optimize.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/clang/test/Interpreter/pragma-optimize.cpp b/clang/test/Interpreter/pragma-optimize.cpp
index 1d43c54e8edb42..580901117da51a 100644
--- a/clang/test/Interpreter/pragma-optimize.cpp
+++ b/clang/test/Interpreter/pragma-optimize.cpp
@@ -3,7 +3,8 @@
// '#pragma clang repl optimize(<opt>)' sets the optimization level and size for
// input parsed after it; <opt> is an -O flag spelled without the leading dash.
//
-// RUN: cat %s | clang-repl -Xcc -Xclang -Xcc -emit-llvm | FileCheck %s
+// RUN: cat %s | clang-repl -Xcc -fno-discard-value-names -Xcc -Xclang -Xcc -emit-llvm \
+// RUN: | FileCheck %s
extern "C" int f_o0(int x) { return x; }
// CHECK: ; Function Attrs: {{.*}}noinline {{.*}}optnone
>From 5916210dbe10049c264ecf0a0ad685d101c392d9 Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin at gmail.com>
Date: Sat, 24 Oct 2026 17:13:59 +0200
Subject: [PATCH 28/37] [mlir] Find the pattern dump root only when IR
debugging is enabled (#233771)
Extracted from https://github.com/llvm/llvm-project/pull/231423
`PatternApplicator` debug printing was unconditionally calling
`getDumpRootOp` which walks nested ops chains. This was taking a
noticeable time in synthetic 1000-nested test. Call `getDumpRootOp` only
if printing is actually enabled.
---
mlir/lib/Rewrite/PatternApplicator.cpp | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/mlir/lib/Rewrite/PatternApplicator.cpp b/mlir/lib/Rewrite/PatternApplicator.cpp
index e1b56fd6efda00..fc415e7dbb3847 100644
--- a/mlir/lib/Rewrite/PatternApplicator.cpp
+++ b/mlir/lib/Rewrite/PatternApplicator.cpp
@@ -200,7 +200,10 @@ LogicalResult PatternApplicator::matchAndRewrite(
#ifndef NDEBUG
// Operation `op` may be invalidated after applying the rewrite
// pattern.
- Operation *dumpRootOp = getDumpRootOp(op);
+ Operation *dumpRootOp = nullptr;
+ // Only find the dump root when level-2 IR debugging is enabled.
+ if (llvm::DebugFlag && llvm::isCurrentDebugType(DEBUG_TYPE, 2))
+ dumpRootOp = getDumpRootOp(op);
#endif
if (pdlMatch) {
result =
>From 127d4f032302e315e03b78f40a535fcad13f7a55 Mon Sep 17 00:00:00 2001
From: Daniel Petrovic <daniel-dev at hotmail.de>
Date: Sat, 24 Oct 2026 17:31:48 +0200
Subject: [PATCH 29/37] =?UTF-8?q?[clang-tidy]=20Don't=20suggest=20const=20?=
=?UTF-8?q?on=20`std::moved`=20locals=20in=20templates=20=E2=80=A6=20(#226?=
=?UTF-8?q?577)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
[clang-tidy] Don't suggest const on std::moved locals in templates
Skip diagnostics in template instantiations for locals that are used as
arguments of std::move.
Reason: for some instantiations the move may be converted to copy so the
variable may look unmutated and flagged, whereas for other template
instantiations the real move semantics may apply.
Fixes: #225395
Assisted by Gemini for constructing the matcher.
---------
Co-authored-by: Zeyi Xu <zeyi2 at nekoarch.cc>
---
.../clang-tidy/misc/ConstCorrectnessCheck.cpp | 30 +++++++++++++++++--
clang-tools-extra/docs/ReleaseNotes.md | 8 +++--
.../misc/const-correctness-templates.cpp | 20 +++++++++++++
3 files changed, 53 insertions(+), 5 deletions(-)
diff --git a/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp b/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
index 90919555d447e8..4532a27962af29 100644
--- a/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
@@ -199,6 +199,25 @@ void ConstCorrectnessCheck::registerMatchers(MatchFinder *Finder) {
}
}
+static bool isStdMoveOperandInTemplate(const VarDecl *Variable,
+ const FunctionDecl *Func,
+ ASTContext &Context) {
+ if (Func->isTemplateInstantiation())
+ if (const Stmt *Body = Func->getBody()) {
+ // Match on the instantiated body where `std::move` is resolved to a
+ // concrete function and the argument references the instantiated
+ // variable.
+ const auto Moves =
+ match(stmt(forEachDescendant(
+ callExpr(callee(functionDecl(hasName("std::move"))),
+ hasArgument(0, ignoringParenImpCasts(declRefExpr(
+ to(equalsNode(Variable)))))))),
+ *Body, Context);
+ return !Moves.empty();
+ }
+ return false;
+}
+
static void addConstFixits(const DiagnosticBuilder &Diag,
const VarDecl *Variable,
const FunctionDecl *Function,
@@ -255,9 +274,6 @@ void ConstCorrectnessCheck::check(const MatchFinder::MatchResult &Result) {
/// dependent. Variables like 'int x = 42;' in a template that can become
/// const emit multiple warnings otherwise.
bool IsNormalVariableInTemplate = Function->isTemplateInstantiation();
- if (IsNormalVariableInTemplate &&
- TemplateDiagnosticsCache.contains(Variable->getBeginLoc()))
- return;
VariableCategory VC = VariableCategory::Value;
const QualType VT = Variable->getType();
@@ -269,6 +285,14 @@ void ConstCorrectnessCheck::check(const MatchFinder::MatchResult &Result) {
ArrayT && ArrayT->getElementType()->isPointerType())
VC = VariableCategory::Pointer;
+ if (IsNormalVariableInTemplate && VC != VariableCategory::Pointer &&
+ isStdMoveOperandInTemplate(Variable, Function, *Result.Context))
+ return;
+
+ if (IsNormalVariableInTemplate &&
+ TemplateDiagnosticsCache.contains(Variable->getBeginLoc()))
+ return;
+
const auto CheckValue = [&]() {
// Offload const-analysis to utility function.
if (isMutated(Variable, LocalScope, Function, Result.Context))
diff --git a/clang-tools-extra/docs/ReleaseNotes.md b/clang-tools-extra/docs/ReleaseNotes.md
index 57ed93042aff2c..037df4cd0937c4 100644
--- a/clang-tools-extra/docs/ReleaseNotes.md
+++ b/clang-tools-extra/docs/ReleaseNotes.md
@@ -261,10 +261,14 @@ infrastructure are described first, followed by tool-specific sections.
- No longer diagnoses variables declared with `decltype(auto)`, where the
suggested `const` does not compile.
-
+
- No longer diagnoses parameters of `main`, whose signature is fixed by the
standard.
+ - No longer diagnoses variables used as operands of `std::move` in template
+ instantiations, where the suggested `const` would break the move for
+ instantiations with real move semantics.
+
- Fixed an infinite loop in {doc}`misc-multiple-inheritance
<clang-tidy/checks/misc/multiple-inheritance>` when checking a class that
inherits from itself or has a circular inheritance graph.
@@ -429,7 +433,7 @@ infrastructure are described first, followed by tool-specific sections.
- Ignored preprocessor directives such as `#endif` that appear immediately
before an enum's closing brace, which previously produced a false positive
and a fix-it that inserted a comma after the directive.
-
+
- Fixed a false positive on empty brace initializers of types with default
member initializers.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
index 98bf27033c3b5f..624159210698c5 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
@@ -6,6 +6,9 @@
// RUN: misc-const-correctness.TransformPointersAsValues: false} \
// RUN: }" -- -fno-delayed-template-parsing
+#include <string>
+#include <utility>
+
template <typename T>
void type_dependent_variables() {
T value = 42;
@@ -27,6 +30,23 @@ void instantiate_template_cases() {
type_dependent_variables<float>();
}
+namespace gh225395 {
+
+template <typename T> void my_swap(T &t1, T &t2) {
+ auto tmp = std::move(t1);
+ t1 = std::move(t2);
+ t2 = std::move(tmp);
+}
+void instantiate() {
+ std::string a;
+ std::string b;
+ my_swap(a, b);
+ int x = 0;
+ int y = 0;
+ my_swap(x, y);
+}
+} // namespace gh225395
+
namespace gh57297{
// The expression to check may not be the dependent operand in a dependent
// operator.
>From bfe094117ba828cd338ac415bf0f7e98f8b8d4c8 Mon Sep 17 00:00:00 2001
From: Federico Bruzzone <federico.bruzzone.i at gmail.com>
Date: Sat, 24 Oct 2026 17:40:41 +0200
Subject: [PATCH 30/37] [mlir][linalg] Handle `DropPadUnitDims` when padding
value is defined inside the pad body (#223712)
`DropPadUnitDims` reused the pad's padding value as-is when building the
replacement `tensor.pad`.
If that value was defined inside the original pad's own body, the new op
ended up referencing it from outside its region, and erasing the
original pad then hit an assertion.
This PR clone the defining op when the padding value lives inside the
pad body.
Fixes #203813
---------
Signed-off-by: Federico Bruzzone <federico.bruzzone.i at gmail.com>
---
.../Linalg/Transforms/DropUnitDims.cpp | 4 ++
.../Dialect/Linalg/drop-unit-extent-dims.mlir | 43 +++++++++++++++++++
2 files changed, 47 insertions(+)
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index 75e6fc9491129e..40dc35ce6babbf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -702,6 +702,10 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
return rewriter.notifyMatchFailure(padOp, "Failed to collapse source");
}
+ // Clone the padding value if it is defined in the region of the pad op.
+ if (paddingVal.getParentBlock() == &padOp.getRegion().front())
+ paddingVal = rewriter.clone(*paddingVal.getDefiningOp())->getResult(0);
+
auto newResultType = RankedTensorType::get(
newResultShape, padOp.getResultType().getElementType());
auto newPadOp = tensor::PadOp::create(
diff --git a/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir b/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir
index 841d0e5f565125..0c289823557fd5 100644
--- a/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir
+++ b/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir
@@ -1032,6 +1032,49 @@ func.func @drop_unit_pad_dynamic_dims(%arg0: tensor<1x?xf32>) -> tensor<1x?xf32>
// -----
+func.func @drop_unit_pad_dims_constant_inside_body(%arg0: tensor<1x28x28x1xf32>) -> tensor<1x32x32x1xf32>
+{
+ %0 = tensor.pad %arg0 low[0, 2, 2, 0] high[0, 2, 2, 0] {
+ ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index):
+ %cst = arith.constant 0.0 : f32
+ tensor.yield %cst : f32
+ } : tensor<1x28x28x1xf32> to tensor<1x32x32x1xf32>
+ return %0 : tensor<1x32x32x1xf32>
+}
+
+// CHECK-LABEL: func @drop_unit_pad_dims_constant_inside_body
+// CHECK: %[[CST:.+]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[COLLAPSE:.+]] = tensor.collapse_shape
+// CHECK-SAME: {{\[}}[0, 1], [2, 3]{{\]}} : tensor<1x28x28x1xf32> into tensor<28x28xf32>
+// CHECK: %[[PADDED:.+]] = tensor.pad %[[COLLAPSE]] low[2, 2] high[2, 2]
+// CHECK: tensor.yield %[[CST]]
+// CHECK: } : tensor<28x28xf32> to tensor<32x32xf32>
+// CHECK: tensor.expand_shape %[[PADDED]]
+// CHECK-SAME: {{\[}}[0, 1], [2, 3]{{\]}} output_shape [1, 32, 32, 1] : tensor<32x32xf32> into tensor<1x32x32x1xf32>
+
+// -----
+
+// The pattern must not modify the IR when there are no unit dims to drop.
+
+func.func @negative_drop_pad_without_unit_dims_constant_inside_body(%arg0: tensor<2xf32>) -> tensor<3xf32>
+{
+ %0 = tensor.pad %arg0 low[0] high[1] {
+ ^bb0(%arg1: index):
+ %cst = arith.constant 0.0 : f32
+ tensor.yield %cst : f32
+ } : tensor<2xf32> to tensor<3xf32>
+ return %0 : tensor<3xf32>
+}
+
+// CHECK-LABEL: func @negative_drop_pad_without_unit_dims_constant_inside_body
+// CHECK-NOT: tensor.collapse_shape
+// CHECK: %[[CST:.+]] = arith.constant 0.000000e+00 : f32
+// CHECK: tensor.pad %{{.*}} low[0] high[1]
+// CHECK: tensor.yield %[[CST]]
+// CHECK-NOT: tensor.expand_shape
+
+// -----
+
func.func @do_not_drop_non_constant_padding(%arg0: tensor<1x1x3x1x1xf32>, %pad: f32) -> tensor<1x2x3x1x3xf32>
{
%c0 = arith.constant 0 : index
>From a98c5c2b79b6afe17e831f4962f7f1fa27e9ad86 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper at sifive.com>
Date: Sat, 24 Oct 2026 09:09:18 -0700
Subject: [PATCH 31/37] [SelectionDAG] Implement SDTCisSameAs in verifyNode.
(#235157)
Assisted-by: Claude
---
llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp
index f5042aef054838..029dd583c2736f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp
@@ -277,8 +277,17 @@ void SDNodeInfo::verifyNode(const SelectionDAG &DAG, const SDNode *N) const {
reportNodeError(DAG, N, SS.str());
}
break;
- case SDTCisSameAs:
+ case SDTCisSameAs: {
+ SDNodeValue OtherVal = GetConstraintValue(C.ConstrainingValIdx);
+ EVT OtherVT = OtherVal.getValueType();
+
+ if (VT != OtherVT) {
+ SS << Val << " must have the same type as " << OtherVal << " ("
+ << OtherVT << "), but has type " << VT;
+ reportNodeError(DAG, N, SS.str());
+ }
break;
+ }
case SDTCisVTSmallerThanOp: {
const auto *VTNode = dyn_cast<VTSDNode>(Val.getValue().getNode());
if (!VTNode) {
>From 45e57b7dc4019258349b2bc1e28e9075ad1014ec Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper at sifive.com>
Date: Sat, 24 Oct 2026 09:09:37 -0700
Subject: [PATCH 32/37] [TargetLowering] Don't call SignBitIsZero for ABDS in
expandABD. (#235134)
---
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 0458f1109f54fb..6f63f9ff908d4e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -11698,8 +11698,8 @@ SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
// abds, or abdu with non-negative operands: if the sub is nsw, abs(sub).
// Otherwise abdu: if the sub is nuw then lhs >= rhs, so just sub.
- bool IsNonNegative = DAG.SignBitIsZero(LHS) && DAG.SignBitIsZero(RHS);
- bool UseSigned = IsSigned || IsNonNegative;
+ bool UseSigned =
+ IsSigned || (DAG.SignBitIsZero(LHS) && DAG.SignBitIsZero(RHS));
if (DAG.willNotOverflowSub(UseSigned, LHS, RHS)) {
SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
>From e339ea8d26be09465cc6cff196ad7a4ddac30194 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper at sifive.com>
Date: Sat, 24 Oct 2026 09:09:59 -0700
Subject: [PATCH 33/37] [DAGCombiner] Delay calls to willNotOverflow and
SignBitIsZero in foldABSToABD. (#235129)
Do the cheaper hasOperation checks first
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 956f0d62a17192..43213a82ca1fe0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -12410,21 +12410,16 @@ SDValue DAGCombiner::foldABSToABD(SDNode *N, const SDLoc &DL) {
// fold (abs (sub nsw x, y)) -> abds(x, y)
// fold (abs (add nsw x, -y)) -> abds(x, y)
- bool AbsOpWillNSW =
- AbsOp0->getFlags().hasNoSignedWrap() ||
- (IsAdd ? DAG.willNotOverflowAdd(/*IsSigned=*/true, Op0, Op1)
- : DAG.willNotOverflowSub(/*IsSigned=*/true, Op0, Op1));
-
// Don't fold this for unsupported types as we lose the NSW handling.
if (hasOperation(ISD::ABDS, VT) && TLI.preferABDSToABSWithNSW(VT) &&
- AbsOpWillNSW)
+ (AbsOp0->getFlags().hasNoSignedWrap() ||
+ (IsAdd ? DAG.willNotOverflowAdd(/*IsSigned=*/true, Op0, Op1)
+ : DAG.willNotOverflowSub(/*IsSigned=*/true, Op0, Op1))))
return CreateZextedAbd(ISD::ABDS);
// fold (abs (sub x, y)) -> abdu(x, y)
- bool AbsOpWillNUW =
- !IsAdd && DAG.SignBitIsZero(Op0) && DAG.SignBitIsZero(Op1);
-
- if (hasOperation(ISD::ABDU, VT) && AbsOpWillNUW)
+ if (!IsAdd && hasOperation(ISD::ABDU, VT) && DAG.SignBitIsZero(Op0) &&
+ DAG.SignBitIsZero(Op1))
return CreateZextedAbd(ISD::ABDU);
return SDValue();
>From cfe60cf9d134a336d08aa74d2f72ca5c7b8d4e4b Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper at sifive.com>
Date: Sat, 24 Oct 2026 09:10:19 -0700
Subject: [PATCH 34/37] [RISCV] Use vector getConstant. NFC (#235152)
It will eventually be legalized to a splat_vector and constant, but we
usually use a vector getConstant so let's be consistent.
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index ebbcf93cce642b..a86115f9b36a95 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13721,8 +13721,7 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, Rs2,
DAG.getUNDEF(MVT::v2i16));
if (IntNo == Intrinsic::riscv_pwmaccsu_i32x2) {
- SDValue Zero = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::v4i16,
- DAG.getConstant(0, DL, XLenVT));
+ SDValue Zero = DAG.getConstant(0, DL, MVT::v4i16);
Rs1 = DAG.getNode(RISCVISD::PZIP, DL, MVT::v4i16, Rs1, Zero);
Rs2 = DAG.getNode(RISCVISD::PZIP, DL, MVT::v4i16, Rs2, Zero);
return DAG.getNode(RISCVISD::PMACCSU_HALVES_00, DL, VT, Acc, Rs1, Rs2);
>From 3736dfed9eaa38651d8f8b8ac67693a7fa5d5402 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper at sifive.com>
Date: Sat, 24 Oct 2026 09:11:21 -0700
Subject: [PATCH 35/37] [SelectionDAG][RISCV][X86] Keep jump table range checks
with cf-protection-branch (#234733)
When the default of a switch is unreachable, SelectionDAGBuilder elides
the jump table range check, except when ARM branch-target-enforcement is
enabled, since an unchecked table branch can be a JOP gadget.
Extend this to the cf-protection-branch module flag used by RISC-V
Zicfilp and X86 IBT. The module flag is merged with Module::Min
behavior, so a value of 0 is treated as not enabled.
Assisted-by: Claude
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 22 ++-
.../CodeGen/RISCV/jumptable-cf-protection.ll | 117 +++++++++++++++
.../CodeGen/X86/jumptable-cf-protection.ll | 142 ++++++++++++++++++
3 files changed, 273 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/jumptable-cf-protection.ll
create mode 100644 llvm/test/CodeGen/X86/jumptable-cf-protection.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 58be54855abdea..48c9eedf7b9946 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -12795,16 +12795,22 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
// JTH->FallthroughUnreachable which will use it to suppress the range
// check.
//
- // However, don't do this if we're doing branch target enforcement,
- // because a table branch _without_ a range check can be a tempting JOP
- // gadget - out-of-bounds inputs that are impossible in correct
- // execution become possible again if an attacker can influence the
- // control flow. So if an attacker doesn't already have a BTI bypass
- // available, we don't want them to be able to get one out of this
- // table branch.
+ // However, don't do this if we're doing branch target enforcement
+ // (e.g. Arm BTI, or the cf-protection-branch module flag used by
+ // RISC-V Zicfilp and X86 IBT), because a table branch _without_ a
+ // range check can be a tempting JOP gadget - out-of-bounds inputs
+ // that are impossible in correct execution become possible again if
+ // an attacker can influence the control flow. So if an attacker
+ // doesn't already have a branch target enforcement bypass available,
+ // we don't want them to be able to get one out of this table branch.
if (FallthroughUnreachable) {
Function &CurFunc = CurMF->getFunction();
- if (!CurFunc.hasFnAttribute("branch-target-enforcement"))
+ bool HasCFBranch = false;
+ if (auto *CFBranch = mdconst::extract_or_null<ConstantInt>(
+ CurFunc.getParent()->getModuleFlag("cf-protection-branch")))
+ HasCFBranch = !CFBranch->isZero();
+ if (!HasCFBranch &&
+ !CurFunc.hasFnAttribute("branch-target-enforcement"))
JTH->FallthroughUnreachable = true;
}
diff --git a/llvm/test/CodeGen/RISCV/jumptable-cf-protection.ll b/llvm/test/CodeGen/RISCV/jumptable-cf-protection.ll
new file mode 100644
index 00000000000000..ae54ee134b5b0b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/jumptable-cf-protection.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 < %s | FileCheck %s --check-prefix=NOCF
+; RUN: sed 's/^;ZEROFLAG //' %s | llc -mtriple=riscv64 | FileCheck %s --check-prefix=NOCF
+; RUN: sed 's/^;BRANCHFLAG //' %s | llc -mtriple=riscv64 | FileCheck %s --check-prefix=CF
+
+; With an unreachable default, the jump table range check can be elided unless
+; cf-protection-branch is enabled. Without the range check, the table branch
+; could be a JOP gadget. A value of 0 means the flag is not enabled, which
+; can happen when linking merges the Min flag.
+
+define i32 @switch_unreachable_default(i32 signext %x) {
+; NOCF-LABEL: switch_unreachable_default:
+; NOCF: # %bb.0: # %entry
+; NOCF-NEXT: lui a1, %hi(.LJTI0_0)
+; NOCF-NEXT: addi a1, a1, %lo(.LJTI0_0)
+; NOCF-NEXT: slli a0, a0, 2
+; NOCF-NEXT: add a0, a1, a0
+; NOCF-NEXT: lw a0, 0(a0)
+; NOCF-NEXT: jr a0
+; NOCF-NEXT: .LBB0_1: # %b0
+; NOCF-NEXT: li a0, 10
+; NOCF-NEXT: ret
+; NOCF-NEXT: .LBB0_2: # %b4
+; NOCF-NEXT: li a0, 54
+; NOCF-NEXT: ret
+; NOCF-NEXT: .LBB0_3: # %b2
+; NOCF-NEXT: li a0, 32
+; NOCF-NEXT: ret
+; NOCF-NEXT: .LBB0_4: # %b3
+; NOCF-NEXT: li a0, 43
+; NOCF-NEXT: ret
+; NOCF-NEXT: .LBB0_5: # %b7
+; NOCF-NEXT: li a0, 87
+; NOCF-NEXT: ret
+; NOCF-NEXT: .LBB0_6: # %b1
+; NOCF-NEXT: li a0, 21
+; NOCF-NEXT: ret
+; NOCF-NEXT: .LBB0_7: # %b5
+; NOCF-NEXT: li a0, 65
+; NOCF-NEXT: ret
+; NOCF-NEXT: .LBB0_8: # %b6
+; NOCF-NEXT: li a0, 76
+; NOCF-NEXT: ret
+;
+; CF-LABEL: switch_unreachable_default:
+; CF: # %bb.0: # %entry
+; CF-NEXT: lpad 0
+; CF-NEXT: li a1, 7
+; CF-NEXT: bltu a1, a0, .LBB0_10
+; CF-NEXT: # %bb.1: # %entry
+; CF-NEXT: slli a0, a0, 2
+; CF-NEXT: lui a1, %hi(.LJTI0_0)
+; CF-NEXT: addi a1, a1, %lo(.LJTI0_0)
+; CF-NEXT: add a0, a1, a0
+; CF-NEXT: lw t2, 0(a0)
+; CF-NEXT: jr t2
+; CF-NEXT: .LBB0_2: # %b0
+; CF-NEXT: li a0, 10
+; CF-NEXT: ret
+; CF-NEXT: .LBB0_3: # %b4
+; CF-NEXT: li a0, 54
+; CF-NEXT: ret
+; CF-NEXT: .LBB0_4: # %b2
+; CF-NEXT: li a0, 32
+; CF-NEXT: ret
+; CF-NEXT: .LBB0_5: # %b3
+; CF-NEXT: li a0, 43
+; CF-NEXT: ret
+; CF-NEXT: .LBB0_6: # %b7
+; CF-NEXT: li a0, 87
+; CF-NEXT: ret
+; CF-NEXT: .LBB0_7: # %b1
+; CF-NEXT: li a0, 21
+; CF-NEXT: ret
+; CF-NEXT: .LBB0_8: # %b5
+; CF-NEXT: li a0, 65
+; CF-NEXT: ret
+; CF-NEXT: .LBB0_9: # %b6
+; CF-NEXT: li a0, 76
+; CF-NEXT: ret
+; CF-NEXT: .LBB0_10: # %def
+entry:
+ switch i32 %x, label %def [
+ i32 0, label %b0
+ i32 1, label %b1
+ i32 2, label %b2
+ i32 3, label %b3
+ i32 4, label %b4
+ i32 5, label %b5
+ i32 6, label %b6
+ i32 7, label %b7
+ ]
+def:
+ unreachable
+b0:
+ ret i32 10
+b1:
+ ret i32 21
+b2:
+ ret i32 32
+b3:
+ ret i32 43
+b4:
+ ret i32 54
+b5:
+ ret i32 65
+b6:
+ ret i32 76
+b7:
+ ret i32 87
+}
+
+;ZEROFLAG !llvm.module.flags = !{!0}
+;ZEROFLAG !0 = !{i32 8, !"cf-protection-branch", i32 0}
+;BRANCHFLAG !llvm.module.flags = !{!0, !1}
+;BRANCHFLAG !0 = !{i32 8, !"cf-protection-branch", i32 1}
+;BRANCHFLAG !1 = !{i32 1, !"cf-branch-label-scheme", !"unlabeled"}
diff --git a/llvm/test/CodeGen/X86/jumptable-cf-protection.ll b/llvm/test/CodeGen/X86/jumptable-cf-protection.ll
new file mode 100644
index 00000000000000..aaf466587f38c5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/jumptable-cf-protection.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=x86_64 < %s | FileCheck %s --check-prefix=NOCF
+; RUN: sed 's/^;ZEROFLAG //' %s | llc -mtriple=x86_64 | FileCheck %s --check-prefix=ZERO
+; RUN: sed 's/^;BRANCHFLAG //' %s | llc -mtriple=x86_64 | FileCheck %s --check-prefix=CF
+
+; With an unreachable default, the jump table range check can be elided unless
+; cf-protection-branch is enabled. Without the range check, the table branch
+; could be a JOP gadget. A value of 0 means the flag is not enabled, which
+; can happen when linking merges the Min flag.
+;
+; X86 currently only checks for the presence of the flag when emitting endbr64
+; and notrack, so the ZERO prefix still sees those, but the range check is
+; elided for a value of 0.
+
+define i32 @switch_unreachable_default(i32 signext %x) {
+; NOCF-LABEL: switch_unreachable_default:
+; NOCF: # %bb.0: # %entry
+; NOCF-NEXT: movl %edi, %eax
+; NOCF-NEXT: jmpq *.LJTI0_0(,%rax,8)
+; NOCF-NEXT: .LBB0_1: # %b0
+; NOCF-NEXT: movl $10, %eax
+; NOCF-NEXT: retq
+; NOCF-NEXT: .LBB0_5: # %b4
+; NOCF-NEXT: movl $54, %eax
+; NOCF-NEXT: retq
+; NOCF-NEXT: .LBB0_3: # %b2
+; NOCF-NEXT: movl $32, %eax
+; NOCF-NEXT: retq
+; NOCF-NEXT: .LBB0_4: # %b3
+; NOCF-NEXT: movl $43, %eax
+; NOCF-NEXT: retq
+; NOCF-NEXT: .LBB0_8: # %b7
+; NOCF-NEXT: movl $87, %eax
+; NOCF-NEXT: retq
+; NOCF-NEXT: .LBB0_2: # %b1
+; NOCF-NEXT: movl $21, %eax
+; NOCF-NEXT: retq
+; NOCF-NEXT: .LBB0_6: # %b5
+; NOCF-NEXT: movl $65, %eax
+; NOCF-NEXT: retq
+; NOCF-NEXT: .LBB0_7: # %b6
+; NOCF-NEXT: movl $76, %eax
+; NOCF-NEXT: retq
+;
+; ZERO-LABEL: switch_unreachable_default:
+; ZERO: # %bb.0: # %entry
+; ZERO-NEXT: endbr64
+; ZERO-NEXT: movl %edi, %eax
+; ZERO-NEXT: notrack jmpq *.LJTI0_0(,%rax,8)
+; ZERO-NEXT: .LBB0_1: # %b0
+; ZERO-NEXT: movl $10, %eax
+; ZERO-NEXT: retq
+; ZERO-NEXT: .LBB0_5: # %b4
+; ZERO-NEXT: movl $54, %eax
+; ZERO-NEXT: retq
+; ZERO-NEXT: .LBB0_3: # %b2
+; ZERO-NEXT: movl $32, %eax
+; ZERO-NEXT: retq
+; ZERO-NEXT: .LBB0_4: # %b3
+; ZERO-NEXT: movl $43, %eax
+; ZERO-NEXT: retq
+; ZERO-NEXT: .LBB0_8: # %b7
+; ZERO-NEXT: movl $87, %eax
+; ZERO-NEXT: retq
+; ZERO-NEXT: .LBB0_2: # %b1
+; ZERO-NEXT: movl $21, %eax
+; ZERO-NEXT: retq
+; ZERO-NEXT: .LBB0_6: # %b5
+; ZERO-NEXT: movl $65, %eax
+; ZERO-NEXT: retq
+; ZERO-NEXT: .LBB0_7: # %b6
+; ZERO-NEXT: movl $76, %eax
+; ZERO-NEXT: retq
+;
+; CF-LABEL: switch_unreachable_default:
+; CF: # %bb.0: # %entry
+; CF-NEXT: endbr64
+; CF-NEXT: cmpl $7, %edi
+; CF-NEXT: ja .LBB0_2
+; CF-NEXT: # %bb.1: # %entry
+; CF-NEXT: movl %edi, %eax
+; CF-NEXT: notrack jmpq *.LJTI0_0(,%rax,8)
+; CF-NEXT: .LBB0_3: # %b0
+; CF-NEXT: movl $10, %eax
+; CF-NEXT: retq
+; CF-NEXT: .LBB0_7: # %b4
+; CF-NEXT: movl $54, %eax
+; CF-NEXT: retq
+; CF-NEXT: .LBB0_5: # %b2
+; CF-NEXT: movl $32, %eax
+; CF-NEXT: retq
+; CF-NEXT: .LBB0_6: # %b3
+; CF-NEXT: movl $43, %eax
+; CF-NEXT: retq
+; CF-NEXT: .LBB0_10: # %b7
+; CF-NEXT: movl $87, %eax
+; CF-NEXT: retq
+; CF-NEXT: .LBB0_4: # %b1
+; CF-NEXT: movl $21, %eax
+; CF-NEXT: retq
+; CF-NEXT: .LBB0_8: # %b5
+; CF-NEXT: movl $65, %eax
+; CF-NEXT: retq
+; CF-NEXT: .LBB0_9: # %b6
+; CF-NEXT: movl $76, %eax
+; CF-NEXT: retq
+; CF-NEXT: .LBB0_2: # %def
+entry:
+ switch i32 %x, label %def [
+ i32 0, label %b0
+ i32 1, label %b1
+ i32 2, label %b2
+ i32 3, label %b3
+ i32 4, label %b4
+ i32 5, label %b5
+ i32 6, label %b6
+ i32 7, label %b7
+ ]
+def:
+ unreachable
+b0:
+ ret i32 10
+b1:
+ ret i32 21
+b2:
+ ret i32 32
+b3:
+ ret i32 43
+b4:
+ ret i32 54
+b5:
+ ret i32 65
+b6:
+ ret i32 76
+b7:
+ ret i32 87
+}
+
+;ZEROFLAG !llvm.module.flags = !{!0}
+;ZEROFLAG !0 = !{i32 8, !"cf-protection-branch", i32 0}
+;BRANCHFLAG !llvm.module.flags = !{!0}
+;BRANCHFLAG !0 = !{i32 8, !"cf-protection-branch", i32 1}
>From 7bcef9facae727c9d2be1e4536bf3de21458968d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sat, 24 Oct 2026 09:11:25 -0700
Subject: [PATCH 36/37] RegAlloc: Only def the subregister regunits of a failed
subregister def (#235307)
After an allocation failure, cleanupFailedDefs added each failed def to
the live ranges of all regunits of the assigned register. A subregister
def only writes the subregister, so the remaining units got a def from
an instruction that does not modify them, which the verifier rejects.
Fixes ran-out-of-registers-errors.ll with EXPENSIVE_CHECKS after
#233796.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/CodeGen/RegAllocBase.cpp | 6 ++++-
.../regalloc-failure-subreg-def-regunits.mir | 26 +++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/regalloc-failure-subreg-def-regunits.mir
diff --git a/llvm/lib/CodeGen/RegAllocBase.cpp b/llvm/lib/CodeGen/RegAllocBase.cpp
index 0741e954bcc150..c6f078a63722fc 100644
--- a/llvm/lib/CodeGen/RegAllocBase.cpp
+++ b/llvm/lib/CodeGen/RegAllocBase.cpp
@@ -216,8 +216,12 @@ void RegAllocBase::cleanupFailedDefs(MachineInstr &MI, Register FailedReg,
// Add the def to the PhysReg regunit ranges so it interferes with later
// assignments. Ranges computed later will include the rewritten def.
+ MCRegister DefReg = PhysReg;
+ if (MO.getSubReg())
+ DefReg = TRI->getSubReg(PhysReg, MO.getSubReg());
+
SlotIndex DefIdx = Idx.getRegSlot(MO.isEarlyClobber());
- for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
+ for (MCRegUnit Unit : TRI->regunits(DefReg)) {
LiveRange *LR = LIS->getCachedRegUnit(Unit);
if (LR && !LR->liveAt(DefIdx))
LR->createDeadDef(DefIdx, LIS->getVNInfoAllocator());
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-failure-subreg-def-regunits.mir b/llvm/test/CodeGen/AMDGPU/regalloc-failure-subreg-def-regunits.mir
new file mode 100644
index 00000000000000..6195d7b6eddc4e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-failure-subreg-def-regunits.mir
@@ -0,0 +1,26 @@
+# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -run-pass=regallocbasic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -run-pass=greedy -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+
+# The failed subregister def should only def the subregister's regunits.
+
+# CHECK: error: <unknown>:0:0: ran out of registers during register allocation in function 'subreg_def'
+
+---
+name: subreg_def
+tracksRegLiveness: true
+machineFunctionInfo:
+ scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
+ stackPtrOffsetReg: '$sgpr32'
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+
+ undef %0.sub1:vreg_64 = COPY $vgpr1
+ %0.sub0:vreg_64 = COPY $vgpr0
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD %0, 0, 0, implicit $exec :: (volatile load (s32), addrspace 1)
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD %0, 0, 0, implicit $exec :: (volatile load (s32), addrspace 1)
+ KILL %0.sub0, %0.sub1
+ %3:vgpr_32 = V_ADD_U32_e32 %1, %2, implicit $exec
+ $vgpr0 = COPY %3
+ SI_RETURN implicit $vgpr0
+...
>From 42e2c69004a2c1a83b64e818ddd9ba49296eb1f0 Mon Sep 17 00:00:00 2001
From: Lexi Bromfield <lexinadia at google.com>
Date: Sat, 24 Oct 2026 16:38:19 +0000
Subject: [PATCH 37/37] Remove duplicate PluginPlatformEmscripten target
post-merge
---
.../lldb/source/Plugins/BUILD.bazel | 15 ---------------
1 file changed, 15 deletions(-)
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index c7bbe95c9c1b37..9618ec16e8ede2 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -640,21 +640,6 @@ cc_library(
],
)
-cc_library(
- name = "PluginPlatformEmscripten",
- srcs = glob(["Platform/Emscripten/*.cpp"]),
- hdrs = glob(["Platform/Emscripten/*.h"]),
- includes = [".."],
- deps = [
- ":PluginPlatformPOSIX",
- "//lldb:Core",
- "//lldb:Headers",
- "//lldb:Host",
- "//lldb:Target",
- "//lldb:Utility",
- ],
-)
-
cc_library(
name = "PluginPlatformFreeBSD",
srcs = glob(["Platform/FreeBSD/*.cpp"]),
More information about the libcxx-commits
mailing list