[Mlir-commits] [llvm] [mlir] [mlir] Add uniformity analysis and InferUniformityOpInterface (PR #220740)
Alessandro Potenza
llvmlistbot at llvm.org
Wed Sep 2 14:51:13 PDT 2026
https://github.com/alepot55 updated https://github.com/llvm/llvm-project/pull/220740
>From 8357e77e169b160e08c3b9be7c3fb4d5c16dcefc Mon Sep 17 00:00:00 2001
From: Alessandro Potenza <ap.alessandro.potenza at gmail.com>
Date: Wed, 2 Sep 2026 15:55:26 +0200
Subject: [PATCH 1/2] [mlir][dataflow] Make visitRegionSuccessors overridable
The method is virtual and its comment says it may be overridden to control
precisely how the region successors of a branch are visited, but it is declared
in the private section, so an override cannot delegate to the default
implementation for the cases it does not want to handle itself.
Move the declaration to the protected section, next to the other hooks a
subclass is meant to override. NFC: only the access level changes. The twin in
AbstractSparseBackwardDataFlowAnalysis is left alone because it is not virtual,
so making it protected would not let an override delegate to it.
---
.../mlir/Analysis/DataFlow/SparseAnalysis.h | 28 +++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
index df50d8d193aeb..fa313ae7a4440 100644
--- a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
+++ b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
@@ -257,6 +257,20 @@ class AbstractSparseForwardDataFlowAnalysis : public DataFlowAnalysis {
visitCallableOperation(CallableOpInterface callable,
ArrayRef<AbstractSparseLattice *> argLattices);
+ /// Visit a program point `point` with predecessors within a region branch
+ /// operation `branch`, which can either be the entry block of one of the
+ /// regions or the parent operation itself, and set either the argument or
+ /// parent result lattices.
+ /// This method can be overridden to control precisely how the region
+ /// successors of `branch` are visited. For example in order to precisely
+ /// control the order in which predecessor operand lattices are propagated
+ /// from. An override is responsible for visiting all the known predecessors
+ /// and propagating therefrom.
+ virtual void
+ visitRegionSuccessors(ProgramPoint *point, RegionBranchOpInterface branch,
+ RegionSuccessor successor,
+ ArrayRef<AbstractSparseLattice *> lattices);
+
private:
/// Recursively initialize the analysis on nested operations and blocks.
LogicalResult initializeRecursively(Operation *op);
@@ -272,20 +286,6 @@ class AbstractSparseForwardDataFlowAnalysis : public DataFlowAnalysis {
/// region terminators or callable callsites. Otherwise, the values are
/// determined from block predecessors.
void visitBlock(Block *block);
-
- /// Visit a program point `point` with predecessors within a region branch
- /// operation `branch`, which can either be the entry block of one of the
- /// regions or the parent operation itself, and set either the argument or
- /// parent result lattices.
- /// This method can be overridden to control precisely how the region
- /// successors of `branch` are visited. For example in order to precisely
- /// control the order in which predecessor operand lattices are propagated
- /// from. An override is responsible for visiting all the known predecessors
- /// and propagating therefrom.
- virtual void
- visitRegionSuccessors(ProgramPoint *point, RegionBranchOpInterface branch,
- RegionSuccessor successor,
- ArrayRef<AbstractSparseLattice *> lattices);
};
//===----------------------------------------------------------------------===//
>From 721f0e80e82db5eaf3fb89d27b1303aa2e8ff226 Mon Sep 17 00:00:00 2001
From: Alessandro Potenza <ap.alessandro.potenza at gmail.com>
Date: Wed, 2 Sep 2026 23:49:41 +0200
Subject: [PATCH 2/2] [mlir] Add uniformity analysis and
InferUniformityOpInterface
A sparse forward dataflow analysis that computes, for each SSA value, the widest
group of threads of a SIMT launch (subgroup, workgroup, cluster, the whole
launch) within which every thread observes the same value, and a query for the
widest group that executes an operation together.
Operations describe how uniformity flows through them with
InferUniformityOpInterface. The GPU dialect implements it for thread identity,
launch configuration, collectives, gpu.launch, gpu.func and
gpu.warp_execute_on_lane_0; scf.for and affine.for implement it for their
induction variable. A memory-effect-free operation without regions of a core
dialect is the join of its operands; any other operation without the interface
defines divergent values.
Region control flow needs no per-operation knowledge: the results of a region
branch are joined with the uniformity of the operands that steer it, which are
exactly the operands RegionBranchOpInterface does not forward to a successor.
The analysis assumes structured reconvergence.
---
.../Analysis/DataFlow/UniformityAnalysis.h | 183 ++++++++
.../mlir/Dialect/Affine/IR/AffineOps.h | 1 +
.../mlir/Dialect/Affine/IR/AffineOps.td | 4 +-
mlir/include/mlir/Dialect/GPU/IR/GPUDialect.h | 1 +
mlir/include/mlir/Dialect/GPU/IR/GPUOps.td | 39 +-
mlir/include/mlir/Dialect/SCF/IR/SCF.h | 1 +
mlir/include/mlir/Dialect/SCF/IR/SCFOps.td | 2 +
mlir/include/mlir/Interfaces/CMakeLists.txt | 1 +
.../Interfaces/InferUniformityOpInterface.h | 117 +++++
.../Interfaces/InferUniformityOpInterface.td | 76 ++++
mlir/lib/Analysis/CMakeLists.txt | 2 +
.../Analysis/DataFlow/UniformityAnalysis.cpp | 400 ++++++++++++++++++
mlir/lib/Dialect/Affine/IR/AffineOps.cpp | 12 +
mlir/lib/Dialect/Affine/IR/CMakeLists.txt | 1 +
mlir/lib/Dialect/GPU/CMakeLists.txt | 2 +
.../IR/InferUniformityOpInterfaceImpls.cpp | 188 ++++++++
mlir/lib/Dialect/SCF/IR/CMakeLists.txt | 1 +
mlir/lib/Dialect/SCF/IR/SCF.cpp | 10 +
mlir/lib/Interfaces/CMakeLists.txt | 2 +
.../Interfaces/InferUniformityOpInterface.cpp | 41 ++
.../DataFlow/test-uniformity-analysis.mlir | 367 ++++++++++++++++
mlir/test/lib/Analysis/CMakeLists.txt | 1 +
.../DataFlow/TestUniformityAnalysis.cpp | 66 +++
mlir/tools/mlir-opt/mlir-opt.cpp | 2 +
.../llvm-project-overlay/mlir/BUILD.bazel | 37 ++
25 files changed, 1545 insertions(+), 12 deletions(-)
create mode 100644 mlir/include/mlir/Analysis/DataFlow/UniformityAnalysis.h
create mode 100644 mlir/include/mlir/Interfaces/InferUniformityOpInterface.h
create mode 100644 mlir/include/mlir/Interfaces/InferUniformityOpInterface.td
create mode 100644 mlir/lib/Analysis/DataFlow/UniformityAnalysis.cpp
create mode 100644 mlir/lib/Dialect/GPU/IR/InferUniformityOpInterfaceImpls.cpp
create mode 100644 mlir/lib/Interfaces/InferUniformityOpInterface.cpp
create mode 100644 mlir/test/Analysis/DataFlow/test-uniformity-analysis.mlir
create mode 100644 mlir/test/lib/Analysis/DataFlow/TestUniformityAnalysis.cpp
diff --git a/mlir/include/mlir/Analysis/DataFlow/UniformityAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/UniformityAnalysis.h
new file mode 100644
index 0000000000000..c1b1596a4f7d2
--- /dev/null
+++ b/mlir/include/mlir/Analysis/DataFlow/UniformityAnalysis.h
@@ -0,0 +1,183 @@
+//===- UniformityAnalysis.h - Uniformity analysis ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the dataflow analysis that computes, for SIMT programs
+// expressed with structured control flow, the uniformity of every SSA value:
+// the widest group of threads within which all threads observe the same value.
+// Operations participate in the analysis by implementing
+// `InferUniformityOpInterface`.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_ANALYSIS_DATAFLOW_UNIFORMITYANALYSIS_H
+#define MLIR_ANALYSIS_DATAFLOW_UNIFORMITYANALYSIS_H
+
+#include "mlir/Analysis/DataFlow/SparseAnalysis.h"
+#include "mlir/Interfaces/InferUniformityOpInterface.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/StringSet.h"
+
+namespace mlir::dataflow {
+
+/// The uniformity lattice element of an SSA value.
+class UniformityLattice : public Lattice<Uniformity> {
+public:
+ using Lattice::Lattice;
+};
+
+/// Uniformity analysis determines, for each SSA value, the widest group of
+/// threads (subgroup, workgroup, cluster, the whole launch) within which every
+/// thread observes the same value. A value that two threads of the same
+/// subgroup may observe differently is divergent.
+///
+/// A value `v` is uniform within a group at a program point `p` if all threads
+/// of the group that execute the dynamic instance of `p` together observe the
+/// same `v`. The analysis only reports uniformity that holds regardless of
+/// which threads are active, so its facts stay true when a value is hoisted
+/// out of divergent control flow.
+///
+/// Transfer functions:
+///
+/// - An operation implementing `InferUniformityOpInterface` describes the
+/// uniformity of the values it defines itself.
+/// - A memory-effect-free operation without regions of a *transparent*
+/// dialect computes a function of its operands, so its results are the join
+/// of the operands (uniform when it has none). The default set of
+/// transparent dialects is the one of the core dialects that have no notion
+/// of a thread.
+/// - Any other operation defines divergent values: an operation that reads
+/// memory, an operation whose regions may capture values from above without
+/// region control flow (a `linalg.generic`), and an operation of a dialect
+/// that may read thread identity without saying so through the interface.
+/// - The results of an operation with region control flow are the join of the
+/// values forwarded to them by region control flow, joined with the
+/// uniformity of the operands that steer that control flow (the condition
+/// of an `scf.if`, the bounds of an `scf.for`, the condition of an
+/// `scf.condition`): threads that take different paths reach the results
+/// with different values, and a loop whose trip count varies across threads
+/// exits with different values. The entry block arguments of the regions
+/// are only the join of the forwarded values, since they are observed by
+/// the threads that entered the region together.
+/// - In a region with unstructured control flow, the arguments of every
+/// non-entry block are joined with the uniformity of the operands that
+/// steer every branch of the region.
+///
+/// The analysis assumes structured reconvergence: all threads of a group that
+/// execute an operation with region control flow together reconverge after
+/// it. This analysis depends on DeadCodeAnalysis and will be a silent no-op if
+/// DeadCodeAnalysis is not loaded in the same solver context.
+class UniformityAnalysis
+ : public SparseForwardDataFlowAnalysis<UniformityLattice> {
+public:
+ /// Creates the analysis. `transparentDialects` lists the dialects whose
+ /// memory-effect-free operations are known to compute a function of their
+ /// operands only; it defaults to `getDefaultTransparentDialects()`.
+ explicit UniformityAnalysis(DataFlowSolver &solver,
+ ArrayRef<StringRef> transparentDialects =
+ getDefaultTransparentDialects());
+
+ /// The dialects treated as transparent by default: `affine`, `arith`,
+ /// `bufferization`, `builtin`, `cf`, `complex`, `func`, `index`, `linalg`,
+ /// `math`, `memref`, `scf`, `tensor`, `ub` and `vector`.
+ static ArrayRef<StringRef> getDefaultTransparentDialects();
+
+ LogicalResult initialize(Operation *top) override;
+
+ LogicalResult visit(ProgramPoint *point) override;
+
+ /// At an entry point, a value is divergent.
+ void setToEntryState(UniformityLattice *lattice) override;
+
+ LogicalResult visitOperation(Operation *op,
+ ArrayRef<const UniformityLattice *> operands,
+ ArrayRef<UniformityLattice *> results) override;
+
+ /// Infers block arguments and results that region control flow does not
+ /// forward a value to (loop induction variables, the body arguments of
+ /// `gpu.launch`) through `InferUniformityOpInterface`.
+ void visitNonControlFlowArguments(
+ Operation *op, const RegionSuccessor &successor,
+ ValueRange nonSuccessorInputs,
+ ArrayRef<UniformityLattice *> nonSuccessorInputLattices) override;
+
+protected:
+ /// Lets a callable implementing `InferUniformityOpInterface` (a GPU kernel)
+ /// describe the uniformity of its arguments.
+ void visitCallableOperation(
+ CallableOpInterface callable,
+ ArrayRef<AbstractSparseLattice *> argLattices) override;
+
+ /// Joins the forwarded values as usual, then adds the control dependence on
+ /// the operands steering the region branch; or defers entirely to
+ /// `InferUniformityOpInterface` when the operation implements it for the
+ /// successor being visited.
+ void
+ visitRegionSuccessors(ProgramPoint *point, RegionBranchOpInterface branch,
+ RegionSuccessor successor,
+ ArrayRef<AbstractSparseLattice *> lattices) override;
+
+private:
+ /// Returns true if `op` belongs to a transparent dialect.
+ bool isTransparent(Operation *op) const;
+
+ /// Runs the interface of `op` at `point` and joins what it reports into the
+ /// lattices of the `candidates` it names. Returns which candidates it set,
+ /// and sets `pending` when an operand of `op` is still uninitialized, in
+ /// which case a candidate the interface did not name must be left alone
+ /// rather than put in the entry state.
+ llvm::SmallBitVector inferThroughInterface(
+ InferUniformityOpInterface op, ProgramPoint *point, ValueRange candidates,
+ ArrayRef<AbstractSparseLattice *> lattices, bool &pending);
+
+ /// Puts the lattices that `set` does not mark in the entry state.
+ void setUnsetToEntryStates(const llvm::SmallBitVector &set,
+ ArrayRef<AbstractSparseLattice *> lattices);
+
+ /// Joins the uniformity of the operands steering `branch` and the
+ /// terminators of its regions into `lattices`, the result lattices of
+ /// `branch`.
+ void joinControlDependence(ProgramPoint *point, Operation *branch,
+ ArrayRef<AbstractSparseLattice *> lattices);
+
+ /// Joins the uniformity of the operands steering every branch in the region
+ /// of `block` into the arguments of `block`, if it is not an entry block.
+ void visitUnstructuredBlockArguments(Block *block);
+
+ llvm::StringSet<> transparentDialects;
+};
+
+/// Collects into `controlOperands` the operands of `op` that steer control
+/// flow rather than being forwarded to a successor: the operands of an
+/// operation implementing `RegionBranchOpInterface` that are not entry
+/// successor operands of any region successor (the condition of `scf.if`, the
+/// bounds of `scf.for`), the operands of a `RegionBranchTerminatorOpInterface`
+/// that are not successor operands of any successor (the condition of
+/// `scf.condition`), and the operands of a `BranchOpInterface` that are not
+/// forwarded to any successor (the condition of `cf.cond_br`). Collects nothing
+/// for any other operation.
+void getControlOperands(Operation *op, SmallVectorImpl<Value> &controlOperands);
+
+/// Returns the uniformity scope of `value` after `solver` has run. A value the
+/// analysis did not reach is reported as divergent.
+UniformityScope getUniformity(DataFlowSolver &solver, Value value);
+
+/// Returns the widest group of threads that execute `op` together: the meet of
+/// the uniformity of the operands steering every enclosing region branch and
+/// every branch of a region with unstructured control flow, up to the closest
+/// enclosing callable or launch boundary (see
+/// `InferUniformityOpInterface::isLaunchBoundary`). A `gpu.barrier` whose
+/// execution uniformity is narrower than its scope may deadlock. If
+/// `narrowingOperand` is given, it receives
+/// the first control operand that narrows the execution to the returned scope,
+/// or null when nothing does.
+UniformityScope getExecutionUniformity(DataFlowSolver &solver, Operation *op,
+ Value *narrowingOperand = nullptr);
+
+} // namespace mlir::dataflow
+
+#endif // MLIR_ANALYSIS_DATAFLOW_UNIFORMITYANALYSIS_H
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
index c5cd8c6c31375..83430fe1b68c0 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
@@ -21,6 +21,7 @@
#include "mlir/IR/Builders.h"
#include "mlir/Interfaces/AlignmentAttrInterface.h"
#include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "mlir/Interfaces/InferUniformityOpInterface.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
namespace mlir {
namespace affine {
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index 994df5aef0d9a..04316e6d36e26 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -20,6 +20,7 @@ include "mlir/Interfaces/ControlFlowInterfaces.td"
include "mlir/Interfaces/InferIntDivisibilityOpInterface.td"
include "mlir/Interfaces/InferIntRangeInterface.td"
include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/Interfaces/InferUniformityOpInterface.td"
include "mlir/Interfaces/LoopLikeInterface.td"
include "mlir/Interfaces/SideEffectInterfaces.td"
@@ -139,7 +140,8 @@ def AffineForOp : Affine_Op<"for",
"getLoopUpperBounds", "getYieldedValuesMutable",
"replaceWithAdditionalYields", "getStaticTripCount"]>,
DeclareOpInterfaceMethods<RegionBranchOpInterface,
- ["getEntrySuccessorOperands", "getSuccessorInputs"]>]> {
+ ["getEntrySuccessorOperands", "getSuccessorInputs"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]> {
let summary = "for operation";
let description = [{
Syntax:
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUDialect.h b/mlir/include/mlir/Dialect/GPU/IR/GPUDialect.h
index 5880c78136566..1a67ee7f56537 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUDialect.h
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUDialect.h
@@ -28,6 +28,7 @@
#include "mlir/Interfaces/FunctionInterfaces.h"
#include "mlir/Interfaces/InferIntRangeInterface.h"
#include "mlir/Interfaces/InferTypeOpInterface.h"
+#include "mlir/Interfaces/InferUniformityOpInterface.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "llvm/ADT/STLExtras.h"
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 13fe6d3f25c12..df80f99604fe9 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -29,6 +29,7 @@ include "mlir/IR/OpAsmInterface.td"
include "mlir/Interfaces/FunctionInterfaces.td"
include "mlir/Interfaces/InferIntRangeInterface.td"
include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/Interfaces/InferUniformityOpInterface.td"
include "mlir/Interfaces/SideEffectInterfaces.td"
//===----------------------------------------------------------------------===//
@@ -42,6 +43,7 @@ class GPU_IndexOp<string mnemonic, list<Trait> traits = []> :
GPU_Op<mnemonic, !listconcat(traits, [
Pure,
DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>,
DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>])>,
Arguments<(ins GPU_DimensionAttr:$dimension,
OptionalAttr<IndexAttr>:$upper_bound)>, Results<(outs Index)> {
@@ -231,7 +233,8 @@ def GPU_ThreadIdOp : GPU_IndexOp<"thread_id"> {
}
def GPU_LaneIdOp : GPU_Op<"lane_id", [
- Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>]> {
+ Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]> {
let description = [{
Returns the lane id within the subgroup (warp/wave).
@@ -251,7 +254,8 @@ def GPU_LaneIdOp : GPU_Op<"lane_id", [
}
def GPU_SubgroupIdOp : GPU_Op<"subgroup_id", [
- Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>]>,
+ Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]>,
Arguments<(ins OptionalAttr<IndexAttr>:$upper_bound)>,
Results<(outs Index:$result)> {
let description = [{
@@ -293,7 +297,8 @@ def GPU_GlobalIdOp : GPU_IndexOp<"global_id"> {
def GPU_NumSubgroupsOp : GPU_Op<"num_subgroups", [
- Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>]>,
+ Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]>,
Arguments<(ins OptionalAttr<IndexAttr>:$upper_bound)>,
Results<(outs Index:$result)> {
let description = [{
@@ -314,7 +319,8 @@ def GPU_NumSubgroupsOp : GPU_Op<"num_subgroups", [
}
def GPU_SubgroupSizeOp : GPU_Op<"subgroup_size", [
- Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>]>,
+ Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]>,
Arguments<(ins OptionalAttr<IndexAttr>:$upper_bound)>,
Results<(outs Index:$result)> {
let description = [{
@@ -348,7 +354,8 @@ def GPU_OptionalDimSizeHintAttr : ConfinedAttr<OptionalAttr<DenseI32ArrayAttr>,
def GPU_GPUFuncOp : GPU_Op<"func", [
HasParent<"GPUModuleOp">, AutomaticAllocationScope, SymbolName,
SymbolVisibility, FunctionOpInterface,
- IsolatedFromAbove, AffineScope
+ IsolatedFromAbove, AffineScope,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>
]> {
let summary = "Function executable on a GPU";
@@ -810,6 +817,8 @@ def GPU_LaunchFuncOp :GPU_Op<"launch_func", [
def GPU_LaunchOp : GPU_Op<"launch", [
AffineScope, AutomaticAllocationScope, AttrSizedOperandSegments,
DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface,
+ ["isLaunchBoundary"]>,
GPU_AsyncOpInterface, RecursiveMemoryEffects]>,
Arguments<(ins Variadic<GPU_AsyncToken>:$asyncDependencies,
Index:$gridSizeX, Index:$gridSizeY, Index:$gridSizeZ,
@@ -1178,7 +1187,8 @@ def GPU_AllReduceOperationAttr : EnumAttr<GPU_Dialect, GPU_AllReduceOperation,
"all_reduce_op">;
def GPU_AllReduceOp : GPU_Op<"all_reduce",
- [SameOperandsAndResultType, IsolatedFromAbove]> {
+ [SameOperandsAndResultType, IsolatedFromAbove,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]> {
let summary = "Reduce values among workgroup.";
let description = [{
The `all_reduce` op reduces the value of every work item across a local
@@ -1226,7 +1236,9 @@ def GPU_AllReduceOp : GPU_Op<"all_reduce",
def AnyIntegerOrFloatOr1DVector :
AnyTypeOf<[AnyIntegerOrFloat, FixedVectorOfRankAndType<[1], [AnyIntegerOrFloat]>]>;
-def GPU_SubgroupReduceOp : GPU_Op<"subgroup_reduce", [SameOperandsAndResultType, NoMemoryEffect]> {
+def GPU_SubgroupReduceOp : GPU_Op<"subgroup_reduce",
+ [SameOperandsAndResultType, NoMemoryEffect,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]> {
let summary = "Reduce values among subgroup.";
let description = [{
The `subgroup_reduce` op reduces the values of lanes (work items) across a
@@ -1320,7 +1332,8 @@ def GPU_ShuffleModeAttr : EnumAttr<GPU_Dialect, GPU_ShuffleMode,
"shuffle_mode">;
def GPU_ShuffleOp : GPU_Op<
- "shuffle", [Pure, AllTypesMatch<["value", "shuffleResult"]>]>,
+ "shuffle", [Pure, AllTypesMatch<["value", "shuffleResult"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]>,
Arguments<(ins AnyIntegerOrFloatOr1DVector:$value, I32:$offset, I32:$width,
GPU_ShuffleModeAttr:$mode)>,
Results<(outs AnyIntegerOrFloatOr1DVector:$shuffleResult, I1:$valid)> {
@@ -1389,7 +1402,8 @@ def GPU_ShuffleOp : GPU_Op<
}
def GPU_RotateOp : GPU_Op<
- "rotate", [Pure, AllTypesMatch<["value", "rotateResult"]>]>,
+ "rotate", [Pure, AllTypesMatch<["value", "rotateResult"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]>,
Arguments<(ins AnyIntegerOrFloatOr1DVector:$value,
ConfinedAttr<I32Attr, [IntMinValue<0>]>:$offset,
ConfinedAttr<I32Attr, [IntPowerOf2]>:$width)>,
@@ -3200,6 +3214,7 @@ def GPU_SetCsrPointersOp : GPU_Op<"set_csr_pointers", [GPU_AsyncOpInterface]> {
def GPU_WarpExecuteOnLane0Op : GPU_Op<"warp_execute_on_lane_0",
[DeclareOpInterfaceMethods<RegionBranchOpInterface, [
"areTypesCompatible", "getSuccessorInputs"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>,
SingleBlockImplicitTerminator<"gpu::YieldOp">,
RecursiveMemoryEffects]> {
let summary = "Executes operations in the associated region on thread #0 of a"
@@ -3349,7 +3364,8 @@ def GPU_BroadcastTypeAttr : EnumAttr<GPU_Dialect, GPU_BroadcastType, "broadcast"
def GPU_SubgroupBroadcastOp : GPU_Op<"subgroup_broadcast",
[NoMemoryEffect, AllTypesMatch<["result", "src"]>,
DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
- DeclareOpInterfaceMethods<ConditionallySpeculatable, ["getSpeculatability"]>] #
+ DeclareOpInterfaceMethods<ConditionallySpeculatable, ["getSpeculatability"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>] #
ElementwiseMappable.traits>,
Arguments<(ins AnyType:$src,
Optional<I32>:$lane,
@@ -3376,7 +3392,8 @@ def GPU_SubgroupBroadcastOp : GPU_Op<"subgroup_broadcast",
let hasVerifier = 1;
}
-def GPU_BallotOp : GPU_Op<"ballot", [NoMemoryEffect]>,
+def GPU_BallotOp : GPU_Op<"ballot", [NoMemoryEffect,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>]>,
Arguments<(ins I1:$predicate)>,
Results<(outs AnySignlessInteger:$result)> {
let summary = "Collects predicate values from all threads in a subgroup.";
diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCF.h b/mlir/include/mlir/Dialect/SCF/IR/SCF.h
index 44cbb458d94fe..dbf97a1ce799e 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCF.h
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCF.h
@@ -21,6 +21,7 @@
#include "mlir/Interfaces/ControlFlowInterfaces.h"
#include "mlir/Interfaces/DestinationStyleOpInterface.h"
#include "mlir/Interfaces/InferTypeOpInterface.h"
+#include "mlir/Interfaces/InferUniformityOpInterface.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
#include "mlir/Interfaces/MemorySlotInterfaces.h"
#include "mlir/Interfaces/ParallelCombiningOpInterface.h"
diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
index b0a34989b6433..544c7fc52abbd 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
@@ -19,6 +19,7 @@ include "mlir/IR/RegionKindInterface.td"
include "mlir/Dialect/SCF/IR/DeviceMappingInterface.td"
include "mlir/Interfaces/DestinationStyleOpInterface.td"
include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/Interfaces/InferUniformityOpInterface.td"
include "mlir/Interfaces/MemorySlotInterfaces.td"
include "mlir/Interfaces/ParallelCombiningOpInterface.td"
include "mlir/Interfaces/SideEffectInterfaces.td"
@@ -167,6 +168,7 @@ def ForOp : SCF_Op<"for",
ConditionallySpeculatable,
DeclareOpInterfaceMethods<RegionBranchOpInterface,
["getEntrySuccessorOperands", "getSuccessorInputs"]>,
+ DeclareOpInterfaceMethods<InferUniformityOpInterface>,
DeclareOpInterfaceMethods<PromotableRegionOpInterface>,
SingleBlockImplicitTerminator<"scf::YieldOp">,
RecursiveMemoryEffects]> {
diff --git a/mlir/include/mlir/Interfaces/CMakeLists.txt b/mlir/include/mlir/Interfaces/CMakeLists.txt
index 6461c68423c73..463250a56c136 100644
--- a/mlir/include/mlir/Interfaces/CMakeLists.txt
+++ b/mlir/include/mlir/Interfaces/CMakeLists.txt
@@ -10,6 +10,7 @@ add_mlir_interface(InferIntDivisibilityOpInterface)
add_mlir_interface(InferIntRangeInterface)
add_mlir_interface(InferStridedMetadataInterface)
add_mlir_interface(InferTypeOpInterface)
+add_mlir_interface(InferUniformityOpInterface)
add_mlir_interface(LoopLikeInterface)
add_mlir_interface(MemOpInterfaces)
add_mlir_interface(ParallelCombiningOpInterface)
diff --git a/mlir/include/mlir/Interfaces/InferUniformityOpInterface.h b/mlir/include/mlir/Interfaces/InferUniformityOpInterface.h
new file mode 100644
index 0000000000000..4ff3df6bdf737
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/InferUniformityOpInterface.h
@@ -0,0 +1,117 @@
+//===- InferUniformityOpInterface.h - Uniformity ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the definitions of the uniformity inference interface
+// defined in `InferUniformityOpInterface.td`, and the lattice value it works
+// with.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_INFERUNIFORMITYOPINTERFACE_H
+#define MLIR_INTERFACES_INFERUNIFORMITYOPINTERFACE_H
+
+#include "mlir/IR/OpDefinition.h"
+#include "llvm/ADT/ArrayRef.h"
+#include <optional>
+
+namespace mlir {
+
+/// The widest group of threads of a SIMT launch within which an SSA value is
+/// known to be the same for every thread, from the narrowest to the widest.
+///
+/// The groups nest: a thread belongs to a subgroup (warp, wave), a subgroup to
+/// a workgroup (thread block), a workgroup to a cluster, and every cluster to
+/// the launch. A value that is the same within a group is also the same within
+/// every narrower group, so the scopes are totally ordered and the meet of two
+/// scopes is the narrower one.
+enum class UniformityScope : uint8_t {
+ /// Two threads of the same subgroup may observe different values.
+ Divergent = 0,
+ /// The same for every thread of a subgroup.
+ Subgroup,
+ /// The same for every thread of a workgroup.
+ Workgroup,
+ /// The same for every thread of a cluster of workgroups.
+ Cluster,
+ /// The same for every thread of the launch.
+ Uniform,
+};
+
+/// Returns the narrower of two scopes.
+inline UniformityScope meet(UniformityScope lhs, UniformityScope rhs) {
+ return lhs < rhs ? lhs : rhs;
+}
+
+/// Returns the name of a scope as written in IR and diagnostics.
+StringRef stringifyUniformityScope(UniformityScope scope);
+
+raw_ostream &operator<<(raw_ostream &os, UniformityScope scope);
+
+/// The uniformity lattice value of an SSA value: a scope, or uninitialized
+/// while the analysis has not reached the value yet.
+class Uniformity {
+public:
+ Uniformity() = default;
+ Uniformity(UniformityScope scope) : scope(scope) {}
+
+ static Uniformity getUniform() {
+ return Uniformity(UniformityScope::Uniform);
+ }
+ static Uniformity getDivergent() {
+ return Uniformity(UniformityScope::Divergent);
+ }
+
+ bool isUninitialized() const { return !scope.has_value(); }
+
+ UniformityScope getScope() const {
+ assert(!isUninitialized() && "querying an uninitialized uniformity");
+ return *scope;
+ }
+
+ bool operator==(const Uniformity &rhs) const { return scope == rhs.scope; }
+ bool operator!=(const Uniformity &rhs) const { return !(*this == rhs); }
+
+ /// The join of two lattice values: uninitialized is the neutral element,
+ /// otherwise the narrower scope. The pessimistic fixpoint is `Divergent`.
+ static Uniformity join(const Uniformity &lhs, const Uniformity &rhs) {
+ if (lhs.isUninitialized())
+ return rhs;
+ if (rhs.isUninitialized())
+ return lhs;
+ return Uniformity(meet(*lhs.scope, *rhs.scope));
+ }
+
+ /// The join of a list of values; uninitialized if the list is empty or all
+ /// of its elements are uninitialized.
+ static Uniformity join(ArrayRef<Uniformity> values) {
+ Uniformity result;
+ for (const Uniformity &value : values)
+ result = join(result, value);
+ return result;
+ }
+
+ void print(raw_ostream &os) const;
+
+private:
+ std::optional<UniformityScope> scope;
+};
+
+inline raw_ostream &operator<<(raw_ostream &os, const Uniformity &uniformity) {
+ uniformity.print(os);
+ return os;
+}
+
+/// The callback through which an operation implementing
+/// `InferUniformityOpInterface` reports the uniformity of a value it defines.
+using SetUniformityFn = llvm::function_ref<void(Value, UniformityScope)>;
+
+} // namespace mlir
+
+#include "mlir/Interfaces/InferUniformityOpInterface.h.inc"
+
+#endif // MLIR_INTERFACES_INFERUNIFORMITYOPINTERFACE_H
diff --git a/mlir/include/mlir/Interfaces/InferUniformityOpInterface.td b/mlir/include/mlir/Interfaces/InferUniformityOpInterface.td
new file mode 100644
index 0000000000000..e6da54601f174
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/InferUniformityOpInterface.td
@@ -0,0 +1,76 @@
+//===- InferUniformityOpInterface.td - Uniformity ----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the interface through which operations participate in uniformity
+// (divergence) analysis on SIMT targets.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_INFERUNIFORMITYOPINTERFACE
+#define MLIR_INTERFACES_INFERUNIFORMITYOPINTERFACE
+
+include "mlir/IR/OpBase.td"
+
+def InferUniformityOpInterface : OpInterface<"InferUniformityOpInterface"> {
+ let description = [{
+ Allows operations to participate in uniformity analysis, which determines
+ for each SSA value the widest group of threads (subgroup, workgroup,
+ cluster, the whole launch) within which every thread observes the same
+ value.
+
+ Operations that read thread identity (`gpu.thread_id`), communicate across
+ lanes (`gpu.subgroup_broadcast`, `gpu.shuffle`) or define values for a
+ region executed by a subset of threads (`gpu.warp_execute_on_lane_0`)
+ implement this interface to describe how uniformity flows through them.
+ Operations that compute a function of their operands only need not
+ implement it: the analysis joins the uniformity of the operands.
+ }];
+ let cppNamespace = "::mlir";
+
+ let methods = [
+ InterfaceMethod<[{
+ Infer the uniformity of the values defined by this operation, given the
+ uniformity of its operands, by calling `setUniformity` for each of them.
+
+ The values that may be set are the results of the operation and the
+ entry block arguments of its regions. A value that is not set keeps the
+ uniformity computed by the analysis: for a region entry block argument
+ or a result that region control flow forwards a value to, the join of
+ the forwarded values; for any other value, divergent.
+
+ For an operation implementing `RegionBranchOpInterface`, setting any
+ successor input of a region successor (a result, or an entry block
+ argument of a region) makes the operation authoritative for all
+ successor inputs of that successor: the analysis then does not join the
+ forwarded values into them. This is how an operation whose results are
+ not the values yielded from its regions, such as a broadcast, describes
+ itself.
+
+ Operands whose uniformity is not known yet are passed as uninitialized;
+ `Uniformity::join` treats them as neutral.
+ }],
+ /*retTy=*/"void",
+ /*methodName=*/"inferUniformity",
+ /*args=*/(ins "::llvm::ArrayRef<::mlir::Uniformity>":$operandUniformity,
+ "::mlir::SetUniformityFn":$setUniformity)>,
+
+ InterfaceMethod<[{
+ Returns true if the regions of this operation are the entry of a SIMT
+ launch: the threads executing them are not the threads executing the
+ operation itself, so the control flow enclosing the operation does not
+ narrow the uniformity of their execution. `gpu.launch` returns true.
+ }],
+ /*retTy=*/"bool",
+ /*methodName=*/"isLaunchBoundary",
+ /*args=*/(ins),
+ /*methodBody=*/"",
+ /*defaultImplementation=*/[{ return false; }]>
+ ];
+}
+
+#endif // MLIR_INTERFACES_INFERUNIFORMITYOPINTERFACE
diff --git a/mlir/lib/Analysis/CMakeLists.txt b/mlir/lib/Analysis/CMakeLists.txt
index 2712ab503663e..3547becaa371d 100644
--- a/mlir/lib/Analysis/CMakeLists.txt
+++ b/mlir/lib/Analysis/CMakeLists.txt
@@ -43,6 +43,7 @@ add_mlir_library(MLIRAnalysis
DataFlow/LivenessAnalysis.cpp
DataFlow/SparseAnalysis.cpp
DataFlow/StridedMetadataRangeAnalysis.cpp
+ DataFlow/UniformityAnalysis.cpp
ADDITIONAL_HEADER_DIRS
${MLIR_MAIN_INCLUDE_DIR}/mlir/Analysis
@@ -60,6 +61,7 @@ add_mlir_library(MLIRAnalysis
MLIRInferIntRangeInterface
MLIRInferStridedMetadataInterface
MLIRInferTypeOpInterface
+ MLIRInferUniformityOpInterface
MLIRLoopLikeInterface
MLIRPresburger
MLIRSideEffectInterfaces
diff --git a/mlir/lib/Analysis/DataFlow/UniformityAnalysis.cpp b/mlir/lib/Analysis/DataFlow/UniformityAnalysis.cpp
new file mode 100644
index 0000000000000..08c5338ba7a2c
--- /dev/null
+++ b/mlir/lib/Analysis/DataFlow/UniformityAnalysis.cpp
@@ -0,0 +1,400 @@
+//===- UniformityAnalysis.cpp - Uniformity analysis -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/DataFlow/UniformityAnalysis.h"
+
+#include "mlir/IR/Operation.h"
+#include "mlir/Interfaces/CallInterfaces.h"
+#include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/Support/DebugLog.h"
+
+#define DEBUG_TYPE "uniformity-analysis"
+
+using namespace mlir;
+using namespace mlir::dataflow;
+
+//===----------------------------------------------------------------------===//
+// Control operands
+//===----------------------------------------------------------------------===//
+
+/// Marks in `forwarded` the positions of the operands in `range`.
+static void markForwarded(OperandRange range, llvm::SmallBitVector &forwarded) {
+ if (range.empty())
+ return;
+ unsigned begin = range.getBeginOperandIndex();
+ for (unsigned i = 0, e = range.size(); i < e; ++i)
+ forwarded.set(begin + i);
+}
+
+void mlir::dataflow::getControlOperands(
+ Operation *op, SmallVectorImpl<Value> &controlOperands) {
+ llvm::SmallBitVector forwarded(op->getNumOperands());
+ if (auto branch = dyn_cast<RegionBranchOpInterface>(op)) {
+ SmallVector<RegionSuccessor> successors;
+ branch.getSuccessorRegions(RegionBranchPoint::parent(), successors);
+ for (const RegionSuccessor &successor : successors)
+ markForwarded(branch.getEntrySuccessorOperands(successor), forwarded);
+ } else if (auto terminator =
+ dyn_cast<RegionBranchTerminatorOpInterface>(op)) {
+ SmallVector<Attribute> operandAttrs(op->getNumOperands(), nullptr);
+ SmallVector<RegionSuccessor> successors;
+ terminator.getSuccessorRegions(operandAttrs, successors);
+ for (const RegionSuccessor &successor : successors)
+ markForwarded(terminator.getSuccessorOperands(successor), forwarded);
+ } else if (auto branch = dyn_cast<BranchOpInterface>(op)) {
+ for (unsigned i = 0, e = op->getNumSuccessors(); i < e; ++i)
+ markForwarded(branch.getSuccessorOperands(i).getForwardedOperands(),
+ forwarded);
+ } else {
+ return;
+ }
+ for (auto [index, operand] : llvm::enumerate(op->getOperands()))
+ if (!forwarded.test(index))
+ controlOperands.push_back(operand);
+}
+
+/// Returns the terminator of `block` if it is one of the branch interfaces
+/// that steer control flow, null otherwise.
+static Operation *getBranchTerminator(Block &block) {
+ if (block.empty())
+ return nullptr;
+ Operation *terminator = &block.back();
+ if (isa<BranchOpInterface, RegionBranchTerminatorOpInterface>(terminator))
+ return terminator;
+ return nullptr;
+}
+
+//===----------------------------------------------------------------------===//
+// UniformityAnalysis
+//===----------------------------------------------------------------------===//
+
+static const StringRef kDefaultTransparentDialects[] = {
+ "affine", "arith", "bufferization", "builtin", "cf",
+ "complex", "func", "index", "linalg", "math",
+ "memref", "scf", "tensor", "ub", "vector"};
+
+ArrayRef<StringRef> UniformityAnalysis::getDefaultTransparentDialects() {
+ return kDefaultTransparentDialects;
+}
+
+UniformityAnalysis::UniformityAnalysis(DataFlowSolver &solver,
+ ArrayRef<StringRef> transparentDialects)
+ : SparseForwardDataFlowAnalysis(solver) {
+ for (StringRef dialect : transparentDialects)
+ this->transparentDialects.insert(dialect);
+}
+
+bool UniformityAnalysis::isTransparent(Operation *op) const {
+ return transparentDialects.contains(op->getName().getDialectNamespace());
+}
+
+void UniformityAnalysis::setToEntryState(UniformityLattice *lattice) {
+ propagateIfChanged(lattice, lattice->join(Uniformity::getDivergent()));
+}
+
+LogicalResult UniformityAnalysis::initialize(Operation *top) {
+ if (failed(SparseForwardDataFlowAnalysis::initialize(top)))
+ return failure();
+ // The base class initializes blocks without going through `visit`, so the
+ // control dependence of unstructured block arguments is added here.
+ top->walk([&](Block *block) { visitUnstructuredBlockArguments(block); });
+ return success();
+}
+
+LogicalResult UniformityAnalysis::visit(ProgramPoint *point) {
+ if (failed(SparseForwardDataFlowAnalysis::visit(point)))
+ return failure();
+ if (point->isBlockStart())
+ visitUnstructuredBlockArguments(point->getBlock());
+ return success();
+}
+
+llvm::SmallBitVector UniformityAnalysis::inferThroughInterface(
+ InferUniformityOpInterface op, ProgramPoint *point, ValueRange candidates,
+ ArrayRef<AbstractSparseLattice *> lattices, bool &pending) {
+ assert(candidates.size() == lattices.size() && "size mismatch");
+ SmallVector<Uniformity> operandUniformity =
+ llvm::map_to_vector(op->getOperands(), [&](Value operand) {
+ return getLatticeElementFor(point, operand)->getValue();
+ });
+ // An implementation is allowed to describe nothing while an operand it needs
+ // is still uninitialized, and several do. That is not the same as saying the
+ // value is divergent: the join is a meet, so a lattice put in the entry state
+ // can never recover, and the operand will settle on a later visit.
+ pending = llvm::any_of(operandUniformity, [](const Uniformity &uniformity) {
+ return uniformity.isUninitialized();
+ });
+ llvm::SmallBitVector set(lattices.size());
+ op.inferUniformity(
+ operandUniformity, [&](Value value, UniformityScope scope) {
+ auto it = llvm::find(candidates, value);
+ if (it == candidates.end())
+ return;
+ unsigned index = std::distance(candidates.begin(), it);
+ set.set(index);
+ auto *lattice = static_cast<UniformityLattice *>(lattices[index]);
+ LDBG() << "Inferred " << scope << " for " << value;
+ propagateIfChanged(lattice, lattice->join(Uniformity(scope)));
+ });
+ return set;
+}
+
+void UniformityAnalysis::setUnsetToEntryStates(
+ const llvm::SmallBitVector &set,
+ ArrayRef<AbstractSparseLattice *> lattices) {
+ for (auto [index, lattice] : llvm::enumerate(lattices))
+ if (!set.test(index))
+ setToEntryState(static_cast<UniformityLattice *>(lattice));
+}
+
+LogicalResult
+UniformityAnalysis::visitOperation(Operation *op,
+ ArrayRef<const UniformityLattice *> operands,
+ ArrayRef<UniformityLattice *> results) {
+ if (auto inferrable = dyn_cast<InferUniformityOpInterface>(op)) {
+ SmallVector<AbstractSparseLattice *> resultLattices(results.begin(),
+ results.end());
+ bool pending = false;
+ llvm::SmallBitVector set =
+ inferThroughInterface(inferrable, getProgramPointAfter(op),
+ op->getResults(), resultLattices, pending);
+ if (!pending)
+ setUnsetToEntryStates(set, resultLattices);
+ return success();
+ }
+
+ // Only a memory-effect-free operation of a transparent dialect that has no
+ // region is known to compute a function of its operands. Anything else may
+ // read memory written by another thread, thread identity, or a value its
+ // region captures from above.
+ if (!isTransparent(op) || !isMemoryEffectFree(op) ||
+ op->getNumRegions() != 0) {
+ setAllToEntryStates(results);
+ return success();
+ }
+
+ Uniformity joined = Uniformity::getUniform();
+ if (!operands.empty()) {
+ joined = Uniformity::join(
+ llvm::map_to_vector(operands, [](const UniformityLattice *lattice) {
+ return lattice->getValue();
+ }));
+ if (joined.isUninitialized())
+ return success();
+ }
+ LDBG() << "Joined operands to " << joined << " for "
+ << OpWithFlags(op, OpPrintingFlags().skipRegions());
+ for (UniformityLattice *lattice : results)
+ propagateIfChanged(lattice, lattice->join(joined));
+ return success();
+}
+
+void UniformityAnalysis::visitNonControlFlowArguments(
+ Operation *op, const RegionSuccessor &successor,
+ ValueRange nonSuccessorInputs,
+ ArrayRef<UniformityLattice *> nonSuccessorInputLattices) {
+ if (auto inferrable = dyn_cast<InferUniformityOpInterface>(op)) {
+ ProgramPoint *point =
+ !successor.isRegion()
+ ? getProgramPointAfter(op)
+ : getProgramPointBefore(&successor.getSuccessor()->front());
+ SmallVector<AbstractSparseLattice *> lattices(
+ nonSuccessorInputLattices.begin(), nonSuccessorInputLattices.end());
+ bool pending = false;
+ llvm::SmallBitVector set = inferThroughInterface(
+ inferrable, point, nonSuccessorInputs, lattices, pending);
+ if (pending)
+ return;
+ if (set.any()) {
+ setUnsetToEntryStates(set, lattices);
+ return;
+ }
+ }
+ SparseForwardDataFlowAnalysis::visitNonControlFlowArguments(
+ op, successor, nonSuccessorInputs, nonSuccessorInputLattices);
+}
+
+void UniformityAnalysis::visitCallableOperation(
+ CallableOpInterface callable,
+ ArrayRef<AbstractSparseLattice *> argLattices) {
+ auto inferrable =
+ dyn_cast<InferUniformityOpInterface>(callable.getOperation());
+ Region *body = callable.getCallableRegion();
+ if (inferrable && body && !body->empty()) {
+ Block &entry = body->front();
+ bool pending = false;
+ llvm::SmallBitVector set =
+ inferThroughInterface(inferrable, getProgramPointBefore(&entry),
+ entry.getArguments(), argLattices, pending);
+ if (pending)
+ return;
+ if (set.any()) {
+ setUnsetToEntryStates(set, argLattices);
+ return;
+ }
+ }
+ AbstractSparseForwardDataFlowAnalysis::visitCallableOperation(callable,
+ argLattices);
+}
+
+void UniformityAnalysis::visitRegionSuccessors(
+ ProgramPoint *point, RegionBranchOpInterface branch,
+ RegionSuccessor successor, ArrayRef<AbstractSparseLattice *> lattices) {
+ Operation *op = branch.getOperation();
+ if (auto inferrable = dyn_cast<InferUniformityOpInterface>(op)) {
+ // The operation is authoritative for this successor if it sets one of
+ // the inputs that region control flow would otherwise forward a value to.
+ // Setting only a non-successor input (a loop induction variable) leaves
+ // the forwarded ones to the usual join.
+ ValueRange candidates = point->isBlockStart()
+ ? ValueRange(point->getBlock()->getArguments())
+ : ValueRange(op->getResults());
+ bool pending = false;
+ llvm::SmallBitVector set =
+ inferThroughInterface(inferrable, point, candidates, lattices, pending);
+ if (pending)
+ return;
+ ValueRange successorInputs = branch.getSuccessorInputs(successor);
+ bool authoritative = false;
+ for (auto [index, candidate] : llvm::enumerate(candidates))
+ if (set.test(index) && llvm::is_contained(successorInputs, candidate))
+ authoritative = true;
+ if (authoritative) {
+ setUnsetToEntryStates(set, lattices);
+ return;
+ }
+ } else if (!isTransparent(op)) {
+ // A region operation of an unknown dialect may run its regions with any
+ // subset of threads, and its results may come from anywhere. (The typed
+ // overload of setAllToEntryStates hides the untyped one.)
+ AbstractSparseForwardDataFlowAnalysis::setAllToEntryStates(lattices);
+ return;
+ }
+
+ AbstractSparseForwardDataFlowAnalysis::visitRegionSuccessors(
+ point, branch, successor, lattices);
+ // Threads that take different paths through the regions reach the results
+ // with different values; the entry block arguments of a region are only
+ // observed by the threads that entered it together.
+ if (!point->isBlockStart())
+ joinControlDependence(point, op, lattices);
+}
+
+void UniformityAnalysis::joinControlDependence(
+ ProgramPoint *point, Operation *branch,
+ ArrayRef<AbstractSparseLattice *> lattices) {
+ SmallVector<Value> controlOperands;
+ getControlOperands(branch, controlOperands);
+ for (Region ®ion : branch->getRegions())
+ for (Block &block : region)
+ if (Operation *terminator = getBranchTerminator(block))
+ getControlOperands(terminator, controlOperands);
+
+ Uniformity taint;
+ for (Value operand : controlOperands)
+ taint = Uniformity::join(taint,
+ getLatticeElementFor(point, operand)->getValue());
+ if (taint.isUninitialized())
+ return;
+ LDBG() << "Control dependence " << taint << " for "
+ << OpWithFlags(branch, OpPrintingFlags().skipRegions());
+ for (AbstractSparseLattice *lattice : lattices) {
+ auto *typed = static_cast<UniformityLattice *>(lattice);
+ propagateIfChanged(typed, typed->join(taint));
+ }
+}
+
+void UniformityAnalysis::visitUnstructuredBlockArguments(Block *block) {
+ if (block->isEntryBlock() || block->getNumArguments() == 0)
+ return;
+ ProgramPoint *point = getProgramPointBefore(block);
+ SmallVector<Value> controlOperands;
+ for (Block &other : *block->getParent())
+ if (Operation *terminator = getBranchTerminator(other))
+ if (isa<BranchOpInterface>(terminator))
+ getControlOperands(terminator, controlOperands);
+
+ Uniformity taint;
+ for (Value operand : controlOperands)
+ taint = Uniformity::join(taint,
+ getLatticeElementFor(point, operand)->getValue());
+ if (taint.isUninitialized())
+ return;
+ for (BlockArgument argument : block->getArguments()) {
+ UniformityLattice *lattice = getLatticeElement(argument);
+ propagateIfChanged(lattice, lattice->join(taint));
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Queries
+//===----------------------------------------------------------------------===//
+
+UniformityScope mlir::dataflow::getUniformity(DataFlowSolver &solver,
+ Value value) {
+ const auto *lattice = solver.lookupState<UniformityLattice>(value);
+ if (!lattice || lattice->getValue().isUninitialized())
+ return UniformityScope::Divergent;
+ return lattice->getValue().getScope();
+}
+
+UniformityScope
+mlir::dataflow::getExecutionUniformity(DataFlowSolver &solver, Operation *op,
+ Value *narrowingOperand) {
+ UniformityScope scope = UniformityScope::Uniform;
+ Value narrowing;
+ auto meetControlOperands = [&](Operation *control) {
+ SmallVector<Value> controlOperands;
+ getControlOperands(control, controlOperands);
+ for (Value operand : controlOperands) {
+ UniformityScope operandScope = getUniformity(solver, operand);
+ if (operandScope < scope) {
+ scope = operandScope;
+ narrowing = operand;
+ }
+ }
+ };
+
+ for (Operation *current = op;;) {
+ Block *block = current->getBlock();
+ if (!block)
+ break;
+ Region *region = block->getParent();
+ // Unstructured control flow within the region.
+ if (!block->isEntryBlock())
+ for (Block &other : *region)
+ if (Operation *terminator = getBranchTerminator(other))
+ if (isa<BranchOpInterface>(terminator))
+ meetControlOperands(terminator);
+
+ Operation *parent = region->getParentOp();
+ if (!parent)
+ break;
+ // The body of a launch is executed by the threads of the launch, whatever
+ // control flow the host wrapped the launch in.
+ if (auto inferrable = dyn_cast<InferUniformityOpInterface>(parent);
+ inferrable && inferrable.isLaunchBoundary())
+ break;
+ if (isa<RegionBranchOpInterface>(parent)) {
+ meetControlOperands(parent);
+ for (Region &other : parent->getRegions())
+ for (Block &otherBlock : other)
+ if (Operation *terminator = getBranchTerminator(otherBlock))
+ if (isa<RegionBranchTerminatorOpInterface>(terminator))
+ meetControlOperands(terminator);
+ }
+ if (isa<CallableOpInterface>(parent))
+ break;
+ current = parent;
+ }
+ if (narrowingOperand)
+ *narrowingOperand = narrowing;
+ return scope;
+}
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 9f0734dab3b31..4991e70ef5af7 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -2800,6 +2800,18 @@ bool AffineForOp::matchingBoundOperandList() {
SmallVector<Region *> AffineForOp::getLoopRegions() { return {&getRegion()}; }
+/// The induction variable is a function of the lower bound operands and of
+/// the step, which is an attribute; the upper bound only decides when a thread
+/// stops iterating.
+void AffineForOp::inferUniformity(ArrayRef<Uniformity> operandUniformity,
+ SetUniformityFn setUniformity) {
+ Uniformity iv = Uniformity::getUniform();
+ for (const Uniformity &operand :
+ operandUniformity.take_front(getLowerBoundOperands().size()))
+ iv = Uniformity::join(iv, operand);
+ setUniformity(getInductionVar(), iv.getScope());
+}
+
std::optional<SmallVector<Value>> AffineForOp::getLoopInductionVars() {
return SmallVector<Value>{getInductionVar()};
}
diff --git a/mlir/lib/Dialect/Affine/IR/CMakeLists.txt b/mlir/lib/Dialect/Affine/IR/CMakeLists.txt
index 1caf2fa396797..7aea509d83c48 100644
--- a/mlir/lib/Dialect/Affine/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Affine/IR/CMakeLists.txt
@@ -20,6 +20,7 @@ add_mlir_dialect_library(MLIRAffineDialect
MLIRInferIntDivisibilityOpInterface
MLIRInferIntRangeInterface
MLIRInferTypeOpInterface
+ MLIRInferUniformityOpInterface
MLIRLoopLikeInterface
MLIRMemRefDialect
MLIRShapedOpInterfaces
diff --git a/mlir/lib/Dialect/GPU/CMakeLists.txt b/mlir/lib/Dialect/GPU/CMakeLists.txt
index 547812da0ab97..0b244994809a5 100644
--- a/mlir/lib/Dialect/GPU/CMakeLists.txt
+++ b/mlir/lib/Dialect/GPU/CMakeLists.txt
@@ -1,6 +1,7 @@
add_mlir_dialect_library(MLIRGPUDialect
IR/GPUDialect.cpp
IR/InferIntRangeInterfaceImpls.cpp
+ IR/InferUniformityOpInterfaceImpls.cpp
IR/ValueBoundsOpInterfaceImpl.cpp
ADDITIONAL_HEADER_DIRS
@@ -19,6 +20,7 @@ add_mlir_dialect_library(MLIRGPUDialect
MLIRControlFlowInterfaces
MLIRFunctionInterfaces
MLIRInferIntRangeInterface
+ MLIRInferUniformityOpInterface
MLIRIR
MLIRMathDialect
MLIRMemRefDialect
diff --git a/mlir/lib/Dialect/GPU/IR/InferUniformityOpInterfaceImpls.cpp b/mlir/lib/Dialect/GPU/IR/InferUniformityOpInterfaceImpls.cpp
new file mode 100644
index 0000000000000..c4e74846333d5
--- /dev/null
+++ b/mlir/lib/Dialect/GPU/IR/InferUniformityOpInterfaceImpls.cpp
@@ -0,0 +1,188 @@
+//===- InferUniformityOpInterfaceImpls.cpp - GPU uniformity ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Interfaces/InferUniformityOpInterface.h"
+
+using namespace mlir;
+using namespace mlir::gpu;
+
+//===----------------------------------------------------------------------===//
+// Thread identity and launch configuration
+//===----------------------------------------------------------------------===//
+
+/// The launch configuration is the same for every thread of the launch.
+#define UNIFORM_OP(OpName) \
+ void OpName::inferUniformity(ArrayRef<Uniformity>, \
+ SetUniformityFn setUniformity) { \
+ setUniformity(getResult(), UniformityScope::Uniform); \
+ }
+
+UNIFORM_OP(GridDimOp)
+UNIFORM_OP(BlockDimOp)
+UNIFORM_OP(ClusterDimOp)
+UNIFORM_OP(ClusterDimBlocksOp)
+UNIFORM_OP(NumSubgroupsOp)
+UNIFORM_OP(SubgroupSizeOp)
+
+#undef UNIFORM_OP
+
+/// An index of the current thread differs between threads of a subgroup.
+void ThreadIdOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Divergent);
+}
+
+void GlobalIdOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Divergent);
+}
+
+void LaneIdOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Divergent);
+}
+
+/// The index of the subgroup is the same for every lane of the subgroup.
+void SubgroupIdOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Subgroup);
+}
+
+/// The index of the workgroup is the same for every thread of the workgroup,
+/// whether within the grid or within the cluster.
+void BlockIdOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Workgroup);
+}
+
+void ClusterBlockIdOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Workgroup);
+}
+
+void ClusterIdOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Cluster);
+}
+
+//===----------------------------------------------------------------------===//
+// gpu.launch / gpu.func
+//===----------------------------------------------------------------------===//
+
+static void setDim3(SetUniformityFn setUniformity, KernelDim3 dims,
+ UniformityScope scope) {
+ setUniformity(dims.x, scope);
+ setUniformity(dims.y, scope);
+ setUniformity(dims.z, scope);
+}
+
+/// The body arguments of a launch: sizes are uniform, block indices are the
+/// same within a workgroup, cluster indices within a cluster, thread indices
+/// differ between threads.
+void LaunchOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setDim3(setUniformity, getGridSize(), UniformityScope::Uniform);
+ setDim3(setUniformity, getBlockSize(), UniformityScope::Uniform);
+ setDim3(setUniformity, getBlockIds(), UniformityScope::Workgroup);
+ setDim3(setUniformity, getThreadIds(), UniformityScope::Divergent);
+ if (std::optional<KernelDim3> clusterSize = getClusterSize())
+ setDim3(setUniformity, *clusterSize, UniformityScope::Uniform);
+ if (std::optional<KernelDim3> clusterIds = getClusterIds())
+ setDim3(setUniformity, *clusterIds, UniformityScope::Cluster);
+}
+
+/// Every thread of a launch receives the same kernel arguments. A workgroup
+/// attribution is the same buffer for every thread of the workgroup; a private
+/// attribution is a different buffer for every thread.
+void GPUFuncOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ if (!isKernel())
+ return;
+ for (BlockArgument argument :
+ getBody().getArguments().take_front(getFunctionType().getNumInputs()))
+ setUniformity(argument, UniformityScope::Uniform);
+ for (BlockArgument attribution : getWorkgroupAttributionBBArgs())
+ setUniformity(attribution, UniformityScope::Workgroup);
+ for (BlockArgument attribution : getPrivateAttributions())
+ setUniformity(attribution, UniformityScope::Divergent);
+}
+
+bool LaunchOp::isLaunchBoundary() { return true; }
+
+//===----------------------------------------------------------------------===//
+// Collectives
+//===----------------------------------------------------------------------===//
+
+/// Every thread of the workgroup receives the reduced value.
+void AllReduceOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Workgroup);
+}
+
+/// Every lane of the subgroup receives the reduced value, unless the subgroup
+/// is split into clusters that each reduce separately.
+void SubgroupReduceOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), getClusterSize() ? UniformityScope::Divergent
+ : UniformityScope::Subgroup);
+}
+
+/// A shuffle permutes values across lanes, so a uniform value stays uniform
+/// and a divergent value generally does not become uniform; whether a lane
+/// received a value depends on the lane. The width must be the same on every
+/// lane, but that is a precondition, not something a shuffle establishes.
+void ShuffleOp::inferUniformity(ArrayRef<Uniformity> operandUniformity,
+ SetUniformityFn setUniformity) {
+ Uniformity value = operandUniformity[0];
+ if (!value.isUninitialized())
+ setUniformity(getShuffleResult(), value.getScope());
+ setUniformity(getValid(), UniformityScope::Divergent);
+}
+
+void RotateOp::inferUniformity(ArrayRef<Uniformity> operandUniformity,
+ SetUniformityFn setUniformity) {
+ Uniformity value = operandUniformity[0];
+ if (!value.isUninitialized())
+ setUniformity(getRotateResult(), value.getScope());
+ setUniformity(getValid(), UniformityScope::Divergent);
+}
+
+/// A broadcast is the operation that makes a value uniform across the
+/// subgroup.
+void SubgroupBroadcastOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Subgroup);
+}
+
+/// Every lane of the subgroup receives the same ballot mask.
+void BallotOp::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ setUniformity(getResult(), UniformityScope::Subgroup);
+}
+
+//===----------------------------------------------------------------------===//
+// gpu.warp_execute_on_lane_0
+//===----------------------------------------------------------------------===//
+
+/// The region executes on a single lane, so within it every value is trivially
+/// the same for all the threads executing it. A result whose type equals the
+/// type yielded from the region is broadcast to every lane of the subgroup; a
+/// result of a different type is distributed across the lanes.
+void WarpExecuteOnLane0Op::inferUniformity(ArrayRef<Uniformity>,
+ SetUniformityFn setUniformity) {
+ for (BlockArgument argument : getWarpRegion().getArguments())
+ setUniformity(argument, UniformityScope::Uniform);
+
+ Operation *terminator = getBody()->getTerminator();
+ for (auto [result, yielded] :
+ llvm::zip_equal(getResults(), terminator->getOperands()))
+ setUniformity(result, result.getType() == yielded.getType()
+ ? UniformityScope::Subgroup
+ : UniformityScope::Divergent);
+}
diff --git a/mlir/lib/Dialect/SCF/IR/CMakeLists.txt b/mlir/lib/Dialect/SCF/IR/CMakeLists.txt
index 5e7d10e4a9d6a..f30fc31b75ffd 100644
--- a/mlir/lib/Dialect/SCF/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/SCF/IR/CMakeLists.txt
@@ -16,6 +16,7 @@ add_mlir_dialect_library(MLIRSCFDialect
MLIRDialectUtils
MLIRFunctionInterfaces
MLIRIR
+ MLIRInferUniformityOpInterface
MLIRLoopLikeInterface
MLIRMemorySlotUtils
MLIRSideEffectInterfaces
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 3388fa490f996..715832a3a841a 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -405,6 +405,16 @@ LogicalResult ForOp::verifyRegions() {
return success();
}
+/// The induction variable is `lb + i * step`, the same for every thread that
+/// is still iterating when the lower bound and the step are; the upper bound
+/// only decides when a thread stops.
+void ForOp::inferUniformity(ArrayRef<Uniformity> operandUniformity,
+ SetUniformityFn setUniformity) {
+ Uniformity iv = Uniformity::join(operandUniformity[0], operandUniformity[2]);
+ if (!iv.isUninitialized())
+ setUniformity(getInductionVar(), iv.getScope());
+}
+
std::optional<SmallVector<Value>> ForOp::getLoopInductionVars() {
return SmallVector<Value>{getInductionVar()};
}
diff --git a/mlir/lib/Interfaces/CMakeLists.txt b/mlir/lib/Interfaces/CMakeLists.txt
index d20d290c45c01..163f66e40eacd 100644
--- a/mlir/lib/Interfaces/CMakeLists.txt
+++ b/mlir/lib/Interfaces/CMakeLists.txt
@@ -13,6 +13,7 @@ set(LLVM_OPTIONAL_SOURCES
InferIntRangeInterface.cpp
InferStridedMetadataInterface.cpp
InferTypeOpInterface.cpp
+ InferUniformityOpInterface.cpp
LoopLikeInterface.cpp
MemOpInterfaces.cpp
MemorySlotInterfaces.cpp
@@ -85,6 +86,7 @@ add_mlir_library(MLIRInferStridedMetadataInterface
)
add_mlir_interface_library(InferTypeOpInterface)
+add_mlir_interface_library(InferUniformityOpInterface)
add_mlir_library(MLIRLoopLikeInterface
LoopLikeInterface.cpp
diff --git a/mlir/lib/Interfaces/InferUniformityOpInterface.cpp b/mlir/lib/Interfaces/InferUniformityOpInterface.cpp
new file mode 100644
index 0000000000000..b3a7429274a36
--- /dev/null
+++ b/mlir/lib/Interfaces/InferUniformityOpInterface.cpp
@@ -0,0 +1,41 @@
+//===- InferUniformityOpInterface.cpp - Uniformity inference --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Interfaces/InferUniformityOpInterface.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace mlir;
+
+#include "mlir/Interfaces/InferUniformityOpInterface.cpp.inc"
+
+StringRef mlir::stringifyUniformityScope(UniformityScope scope) {
+ switch (scope) {
+ case UniformityScope::Divergent:
+ return "divergent";
+ case UniformityScope::Subgroup:
+ return "subgroup";
+ case UniformityScope::Workgroup:
+ return "workgroup";
+ case UniformityScope::Cluster:
+ return "cluster";
+ case UniformityScope::Uniform:
+ return "uniform";
+ }
+ llvm_unreachable("unknown uniformity scope");
+}
+
+raw_ostream &mlir::operator<<(raw_ostream &os, UniformityScope scope) {
+ return os << stringifyUniformityScope(scope);
+}
+
+void Uniformity::print(raw_ostream &os) const {
+ if (isUninitialized())
+ os << "<uninitialized>";
+ else
+ os << *scope;
+}
diff --git a/mlir/test/Analysis/DataFlow/test-uniformity-analysis.mlir b/mlir/test/Analysis/DataFlow/test-uniformity-analysis.mlir
new file mode 100644
index 0000000000000..b629d6532b5ef
--- /dev/null
+++ b/mlir/test/Analysis/DataFlow/test-uniformity-analysis.mlir
@@ -0,0 +1,367 @@
+// RUN: mlir-opt -split-input-file -test-uniformity-analysis -verify-diagnostics %s
+
+// Thread identity and launch configuration.
+gpu.module @sources {
+ gpu.func @kernel(%arg: index) kernel {
+ // expected-remark @below {{uniformity of "tid": results = [divergent], execution = uniform}}
+ %tid = gpu.thread_id x {tag = "tid"}
+ // expected-remark @below {{uniformity of "lane": results = [divergent], execution = uniform}}
+ %lane = gpu.lane_id {tag = "lane"}
+ // expected-remark @below {{uniformity of "sg": results = [subgroup], execution = uniform}}
+ %sg = gpu.subgroup_id {tag = "sg"} : index
+ // expected-remark @below {{uniformity of "bid": results = [workgroup], execution = uniform}}
+ %bid = gpu.block_id y {tag = "bid"}
+ // expected-remark @below {{uniformity of "cid": results = [cluster], execution = uniform}}
+ %cid = gpu.cluster_id x {tag = "cid"}
+ // expected-remark @below {{uniformity of "bdim": results = [uniform], execution = uniform}}
+ %bdim = gpu.block_dim x {tag = "bdim"}
+ // expected-remark @below {{uniformity of "sgsize": results = [uniform], execution = uniform}}
+ %sgsize = gpu.subgroup_size {tag = "sgsize"} : index
+ // Kernel arguments are the same for every thread.
+ // expected-remark @below {{uniformity of "arg_use": results = [uniform], execution = uniform}}
+ %arg_use = arith.addi %arg, %arg {tag = "arg_use"} : index
+ // Constants are uniform; arithmetic joins its operands.
+ // expected-remark @below {{uniformity of "c": results = [uniform], execution = uniform}}
+ %c = arith.constant {tag = "c"} 4 : index
+ // expected-remark @below {{uniformity of "tid_c": results = [divergent], execution = uniform}}
+ %tid_c = arith.muli %tid, %c {tag = "tid_c"} : index
+ // expected-remark @below {{uniformity of "bid_c": results = [workgroup], execution = uniform}}
+ %bid_c = arith.addi %bid, %c {tag = "bid_c"} : index
+ // expected-remark @below {{uniformity of "bid_sg": results = [subgroup], execution = uniform}}
+ %bid_sg = arith.addi %bid, %sg {tag = "bid_sg"} : index
+ // expected-remark @below {{uniformity of "bid_cid": results = [workgroup], execution = uniform}}
+ %bid_cid = arith.addi %bid, %cid {tag = "bid_cid"} : index
+ gpu.return
+ }
+}
+
+// -----
+
+// Collectives.
+gpu.module @collectives {
+ gpu.func @kernel(%arg: f32) kernel {
+ %tid = gpu.thread_id x
+ %v = arith.index_cast %tid : index to i32
+ // expected-remark @below {{uniformity of "all": results = [workgroup], execution = uniform}}
+ %all = gpu.all_reduce add %v {} {tag = "all"} : (i32) -> (i32)
+ // expected-remark @below {{uniformity of "sub": results = [subgroup], execution = uniform}}
+ %sub = gpu.subgroup_reduce add %v {tag = "sub"} : (i32) -> (i32)
+ // expected-remark @below {{uniformity of "clustered": results = [divergent], execution = uniform}}
+ %clustered = gpu.subgroup_reduce add %v cluster(size = 4) {tag = "clustered"} : (i32) -> (i32)
+ // expected-remark @below {{uniformity of "bcast": results = [subgroup], execution = uniform}}
+ %bcast = gpu.subgroup_broadcast %v, first_active_lane {tag = "bcast"} : i32
+ %pred = arith.cmpi ult, %v, %v : i32
+ // expected-remark @below {{uniformity of "ballot": results = [subgroup], execution = uniform}}
+ %ballot = gpu.ballot %pred {tag = "ballot"} : i32
+ %w = arith.constant 32 : i32
+ %o = arith.constant 1 : i32
+ // A shuffle of a divergent value is divergent, of a uniform value uniform.
+ // expected-remark @below {{uniformity of "shfl_div": results = [divergent, divergent], execution = uniform}}
+ %s0, %valid0 = gpu.shuffle xor %v, %o, %w {tag = "shfl_div"} : i32
+ // expected-remark @below {{uniformity of "shfl_uni": results = [uniform, divergent], execution = uniform}}
+ %s1, %valid1 = gpu.shuffle xor %w, %o, %w {tag = "shfl_uni"} : i32
+ gpu.return
+ }
+}
+
+// -----
+
+// Structured control flow: the results of a region branch are tainted by the
+// operands steering it, the values inside are not.
+gpu.module @scf {
+ gpu.func @kernel(%n: index, %flag: i1) kernel {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %tid = gpu.thread_id x
+ %cond = arith.cmpi ult, %tid, %n : index
+ // expected-remark @below {{uniformity of "if_div": results = [divergent], execution = uniform}}
+ %r0 = scf.if %cond -> index {
+ // expected-remark @below {{uniformity of "in_then": results = [uniform], execution = divergent}}
+ %a = arith.addi %c1, %c1 {tag = "in_then"} : index
+ scf.yield %a : index
+ } else {
+ scf.yield %c0 : index
+ } {tag = "if_div"}
+ // expected-remark @below {{uniformity of "if_uni": results = [uniform], execution = uniform}}
+ %r1 = scf.if %flag -> index {
+ scf.yield %c1 : index
+ } else {
+ scf.yield %c0 : index
+ } {tag = "if_uni"}
+ // A loop with uniform bounds has uniform induction variable and results.
+ // expected-remark @below {{uniformity of "for_uni": results = [uniform], execution = uniform}}
+ %r2 = scf.for %i = %c0 to %n step %c1 iter_args(%acc = %c0) -> index {
+ // expected-remark @below {{uniformity of "iv_uni": results = [uniform], execution = uniform}}
+ %next = arith.addi %acc, %i {tag = "iv_uni"} : index
+ scf.yield %next : index
+ } {tag = "for_uni"}
+ // A loop whose trip count differs between threads exits with different
+ // values, but its induction variable is the same for the threads that
+ // are still iterating.
+ // expected-remark @below {{uniformity of "for_div": results = [divergent], execution = uniform}}
+ %r3 = scf.for %j = %c0 to %tid step %c1 iter_args(%acc = %c0) -> index {
+ // expected-remark @below {{uniformity of "iv_div_ub": results = [uniform], execution = divergent}}
+ %next = arith.addi %acc, %j {tag = "iv_div_ub"} : index
+ scf.yield %next : index
+ } {tag = "for_div"}
+ // expected-remark @below {{uniformity of "for_div_lb": results = [divergent], execution = uniform}}
+ %r4 = scf.for %k = %tid to %n step %c1 iter_args(%acc = %c0) -> index {
+ // expected-remark @below {{uniformity of "iv_div_lb": results = [divergent], execution = divergent}}
+ %next = arith.addi %acc, %k {tag = "iv_div_lb"} : index
+ scf.yield %next : index
+ } {tag = "for_div_lb"}
+ // The execution uniformity of an operation is the narrowest scope among
+ // the control operands of its enclosing region branches.
+ %sg = gpu.subgroup_id : index
+ %sgcond = arith.cmpi ult, %sg, %n : index
+ scf.if %sgcond {
+ // expected-remark @below {{uniformity of "in_sg_if": results = [], execution = subgroup}}
+ gpu.barrier {tag = "in_sg_if"}
+ scf.if %cond {
+ // expected-remark @below {{uniformity of "nested": results = [], execution = divergent}}
+ gpu.barrier {tag = "nested"}
+ }
+ }
+ gpu.return
+ }
+}
+
+// -----
+
+// scf.while: the condition steers the results.
+gpu.module @while {
+ gpu.func @kernel(%n: index) kernel {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %tid = gpu.thread_id x
+ // expected-remark @below {{uniformity of "while_div": results = [divergent], execution = uniform}}
+ %r = scf.while (%i = %c0) : (index) -> index {
+ %cond = arith.cmpi ult, %i, %tid : index
+ scf.condition(%cond) %i : index
+ } do {
+ ^bb0(%i: index):
+ // expected-remark @below {{uniformity of "in_while": results = [uniform], execution = divergent}}
+ %next = arith.addi %i, %c1 {tag = "in_while"} : index
+ scf.yield %next : index
+ } attributes {tag = "while_div"}
+ // expected-remark @below {{uniformity of "while_uni": results = [uniform], execution = uniform}}
+ %s = scf.while (%i = %c0) : (index) -> index {
+ %cond = arith.cmpi ult, %i, %n : index
+ scf.condition(%cond) %i : index
+ } do {
+ ^bb0(%i: index):
+ %next = arith.addi %i, %c1 : index
+ scf.yield %next : index
+ } attributes {tag = "while_uni"}
+ gpu.return
+ }
+}
+
+// -----
+
+// Memory: a load is divergent unless an operation says otherwise; a load from
+// a thread-private allocation is always divergent.
+gpu.module @memory {
+ gpu.func @kernel(%m: memref<16xf32>) kernel {
+ %c0 = arith.constant 0 : index
+ // expected-remark @below {{uniformity of "load": results = [divergent], execution = uniform}}
+ %v = memref.load %m[%c0] {tag = "load"} : memref<16xf32>
+ // expected-remark @below {{uniformity of "alloca": results = [divergent], execution = uniform}}
+ %p = memref.alloca() {tag = "alloca"} : memref<4xf32>
+ gpu.return
+ }
+}
+
+// -----
+
+// Kernel attributions: a workgroup buffer is shared by the workgroup, a private
+// buffer is per thread, whatever is done with the value.
+gpu.module @attributions {
+ gpu.func @kernel(%n: index)
+ workgroup(%ws: memref<4xf32, #gpu.address_space<workgroup>>)
+ private(%ps: memref<4xf32, #gpu.address_space<private>>) kernel {
+ // expected-remark @below {{uniformity of "ws_ptr": results = [workgroup], execution = uniform}}
+ %a = memref.extract_aligned_pointer_as_index %ws : memref<4xf32, #gpu.address_space<workgroup>> -> index {tag = "ws_ptr"}
+ // expected-remark @below {{uniformity of "ps_ptr": results = [divergent], execution = uniform}}
+ %b = memref.extract_aligned_pointer_as_index %ps : memref<4xf32, #gpu.address_space<private>> -> index {tag = "ps_ptr"}
+ gpu.return
+ }
+}
+
+// -----
+
+// An operation whose region captures a value from above without region
+// control flow is not a function of its operands: it is divergent.
+gpu.module @regions {
+ gpu.func @kernel(%n: index) kernel {
+ %tid = gpu.thread_id x
+ // expected-remark @below {{uniformity of "generate": results = [divergent], execution = uniform}}
+ %t = tensor.generate {
+ ^bb0(%i: index):
+ %v = arith.addi %i, %tid : index
+ tensor.yield %v : index
+ } {tag = "generate"} : tensor<4xindex>
+ gpu.return
+ }
+}
+
+// -----
+
+// Unstructured control flow: block arguments of non-entry blocks are tainted
+// by every branch of the region, and so is their execution.
+gpu.module @cfg {
+ gpu.func @kernel(%n: index, %flag: i1) kernel {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %tid = gpu.thread_id x
+ %cond = arith.cmpi ult, %tid, %n : index
+ cf.cond_br %cond, ^bb1(%c0 : index), ^bb1(%c1 : index)
+ ^bb1(%x: index):
+ // expected-remark @below {{uniformity of "phi": results = [divergent], execution = divergent}}
+ %y = arith.addi %x, %c1 {tag = "phi"} : index
+ gpu.return
+ }
+ gpu.func @uniform_branch(%n: index, %flag: i1) kernel {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ cf.cond_br %flag, ^bb1(%c0 : index), ^bb1(%c1 : index)
+ ^bb1(%x: index):
+ // expected-remark @below {{uniformity of "phi_uni": results = [uniform], execution = uniform}}
+ %y = arith.addi %x, %c1 {tag = "phi_uni"} : index
+ gpu.return
+ }
+}
+
+// -----
+
+// gpu.launch body arguments, and a warp region. The launch is under host
+// control flow, which does not narrow the execution of its body.
+func.func @launch(%n: index, %go: i1) {
+ %c1 = arith.constant 1 : index
+ scf.if %go {
+ gpu.launch blocks(%bx, %by, %bz) in (%gx = %n, %gy = %c1, %gz = %c1)
+ threads(%tx, %ty, %tz) in (%sx = %n, %sy = %c1, %sz = %c1) {
+ // expected-remark @below {{uniformity of "in_launch": results = [], execution = uniform}}
+ gpu.barrier {tag = "in_launch"}
+ gpu.terminator
+ }
+ }
+ gpu.launch blocks(%bx, %by, %bz) in (%gx = %n, %gy = %c1, %gz = %c1)
+ threads(%tx, %ty, %tz) in (%sx = %n, %sy = %c1, %sz = %c1) {
+ // expected-remark @below {{uniformity of "launch_bx": results = [workgroup], execution = uniform}}
+ %a = arith.addi %bx, %c1 {tag = "launch_bx"} : index
+ // expected-remark @below {{uniformity of "launch_tx": results = [divergent], execution = uniform}}
+ %b = arith.addi %tx, %c1 {tag = "launch_tx"} : index
+ // expected-remark @below {{uniformity of "launch_gx": results = [uniform], execution = uniform}}
+ %c = arith.addi %gx, %c1 {tag = "launch_gx"} : index
+ gpu.terminator
+ }
+ return
+}
+
+func.func @warp(%laneid: index, %v: vector<32xf32>) {
+ // expected-remark @below {{uniformity of "warp": results = [subgroup, divergent], execution = uniform}}
+ %r:2 = gpu.warp_execute_on_lane_0(%laneid)[32] args(%v : vector<32xf32>) -> (f32, vector<1xf32>) {
+ ^bb0(%arg: vector<32xf32>):
+ // expected-remark @below {{uniformity of "in_warp": results = [uniform], execution = divergent}}
+ %s = vector.reduction <add>, %arg {tag = "in_warp"} : vector<32xf32> into f32
+ gpu.yield %s, %arg : f32, vector<32xf32>
+ } {tag = "warp"}
+ return
+}
+
+// -----
+
+// affine.for behaves like scf.for: the induction variable is the join of the
+// lower bound operands, and the upper bound only decides when a thread stops.
+gpu.module @affine {
+ gpu.func @kernel(%n: index) kernel {
+ %tid = gpu.thread_id x
+ affine.for %i = 0 to 8 {
+ // expected-remark @below {{uniformity of "affine_iv": results = [uniform], execution = uniform}}
+ %a = arith.addi %i, %i {tag = "affine_iv"} : index
+ }
+ affine.for %i = 0 to %tid {
+ // The trip count varies across threads, but every thread still in the
+ // loop is on the same iteration.
+ // expected-remark @below {{uniformity of "affine_iv_div_ub": results = [uniform], execution = divergent}}
+ %a = arith.addi %i, %i {tag = "affine_iv_div_ub"} : index
+ }
+ affine.for %i = %tid to 8 {
+ // expected-remark @below {{uniformity of "affine_iv_div_lb": results = [divergent], execution = divergent}}
+ %a = arith.addi %i, %i {tag = "affine_iv_div_lb"} : index
+ }
+ gpu.return
+ }
+}
+
+// -----
+
+// A divergent step taints the induction variable, like a divergent lower bound.
+gpu.module @divergent_step {
+ gpu.func @kernel(%n: index) kernel {
+ %c0 = arith.constant 0 : index
+ %tid = gpu.thread_id x
+ %r = scf.for %i = %c0 to %n step %tid iter_args(%a = %c0) -> (index) {
+ // expected-remark @below {{uniformity of "step_iv": results = [divergent], execution = divergent}}
+ %b = arith.addi %i, %a {tag = "step_iv"} : index
+ scf.yield %b : index
+ }
+ // expected-remark @below {{uniformity of "step_result": results = [divergent], execution = uniform}}
+ %use = arith.addi %r, %r {tag = "step_result"} : index
+ gpu.return
+ }
+}
+
+// -----
+
+// A value an implementation could not decide yet, because an operand had not
+// been visited, must not be pinned to divergent: the loop result below is only
+// known after the regions of the while have settled, and the shuffle that uses
+// it is visited first.
+gpu.module @late_operand {
+ gpu.func @kernel(%n: index) kernel {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %r = scf.while (%i = %c0) : (index) -> index {
+ %c = arith.cmpi ult, %i, %n : index
+ scf.condition(%c) %i : index
+ } do {
+ ^bb0(%i: index):
+ %nx = arith.addi %i, %c1 : index
+ scf.yield %nx : index
+ }
+ %w = arith.index_cast %r : index to i32
+ %off = arith.constant 1 : i32
+ %width = arith.constant 32 : i32
+ // The shuffled value is the loop result, which is only known once the
+ // regions of the while have settled. Pinning it to divergent on the first
+ // visit would be permanent, since the join is a meet.
+ // expected-remark @below {{uniformity of "late_shuffle": results = [uniform, divergent], execution = uniform}}
+ %s, %v = gpu.shuffle xor %w, %off, %width {tag = "late_shuffle"} : i32
+ gpu.return
+ }
+}
+
+// -----
+
+// A callable the analysis is not told about receives divergent arguments: it
+// joins every call site, and nothing says a function is only called from
+// uniform control flow. Execution uniformity stops at the callable, so the
+// body of a callee is reported as executed by the whole launch whatever the
+// call site.
+gpu.module @calls {
+ func.func @callee(%x: index) -> index {
+ // expected-remark @below {{uniformity of "callee_arg": results = [divergent], execution = uniform}}
+ %a = arith.addi %x, %x {tag = "callee_arg"} : index
+ return %a : index
+ }
+ gpu.func @kernel(%n: index) kernel {
+ %tid = gpu.thread_id x
+ // expected-remark @below {{uniformity of "call_uniform": results = [divergent], execution = uniform}}
+ %r1 = func.call @callee(%n) {tag = "call_uniform"} : (index) -> index
+ // expected-remark @below {{uniformity of "call_divergent": results = [divergent], execution = uniform}}
+ %r2 = func.call @callee(%tid) {tag = "call_divergent"} : (index) -> index
+ gpu.return
+ }
+}
diff --git a/mlir/test/lib/Analysis/CMakeLists.txt b/mlir/test/lib/Analysis/CMakeLists.txt
index d86af5017f24b..1b69f7aa9fd31 100644
--- a/mlir/test/lib/Analysis/CMakeLists.txt
+++ b/mlir/test/lib/Analysis/CMakeLists.txt
@@ -19,6 +19,7 @@ add_mlir_library(MLIRTestAnalysis
DataFlow/TestLivenessAnalysis.cpp
DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
DataFlow/TestStridedMetadataRangeAnalysis.cpp
+ DataFlow/TestUniformityAnalysis.cpp
EXCLUDE_FROM_LIBMLIR
diff --git a/mlir/test/lib/Analysis/DataFlow/TestUniformityAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestUniformityAnalysis.cpp
new file mode 100644
index 0000000000000..c6dbb9bc99f92
--- /dev/null
+++ b/mlir/test/lib/Analysis/DataFlow/TestUniformityAnalysis.cpp
@@ -0,0 +1,66 @@
+//===- TestUniformityAnalysis.cpp - Test uniformity analysis --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/DataFlow/UniformityAnalysis.h"
+#include "mlir/Analysis/DataFlow/Utils.h"
+#include "mlir/Analysis/DataFlowFramework.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassRegistry.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace mlir;
+using namespace mlir::dataflow;
+
+namespace {
+/// Reports the uniformity of the results of every operation carrying a `tag`
+/// string attribute, and the uniformity of its execution, as a remark.
+struct TestUniformityAnalysisPass
+ : public PassWrapper<TestUniformityAnalysisPass, OperationPass<>> {
+ MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestUniformityAnalysisPass)
+
+ StringRef getArgument() const override { return "test-uniformity-analysis"; }
+ StringRef getDescription() const override {
+ return "Test uniformity analysis by reporting the uniformity of the "
+ "results and of the execution of every operation with a `tag` "
+ "attribute";
+ }
+
+ void runOnOperation() override {
+ Operation *rootOp = getOperation();
+
+ DataFlowSolver solver;
+ loadBaselineAnalyses(solver);
+ solver.load<UniformityAnalysis>();
+ if (failed(solver.initializeAndRun(rootOp)))
+ return signalPassFailure();
+
+ rootOp->walk([&](Operation *op) {
+ auto tag = op->getAttrOfType<StringAttr>("tag");
+ if (!tag)
+ return;
+ std::string message;
+ llvm::raw_string_ostream os(message);
+ os << "uniformity of \"" << tag.getValue() << "\": results = [";
+ llvm::interleaveComma(op->getResults(), os, [&](Value result) {
+ os << getUniformity(solver, result);
+ });
+ os << "], execution = " << getExecutionUniformity(solver, op);
+ op->emitRemark() << message;
+ });
+ }
+};
+} // end anonymous namespace
+
+namespace mlir::test {
+void registerTestUniformityAnalysisPass() {
+ PassRegistration<TestUniformityAnalysisPass>();
+}
+} // end namespace mlir::test
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 47dffb1b2d28b..8854cf9b35da9 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -158,6 +158,7 @@ void registerTestTensorTransforms();
void registerTestTopologicalSortAnalysisPass();
void registerTestTransformDialectEraseSchedulePass();
void registerTestPassStateExtensionCommunication();
+void registerTestUniformityAnalysisPass();
void registerTestVectorLowerings();
void registerTestVectorReductionToSPIRVDotProd();
void registerTestVulkanRunnerPipeline();
@@ -307,6 +308,7 @@ static void registerTestPasses() {
mlir::test::registerTestTopologicalSortAnalysisPass();
mlir::test::registerTestTransformDialectEraseSchedulePass();
mlir::test::registerTestPassStateExtensionCommunication();
+ mlir::test::registerTestUniformityAnalysisPass();
mlir::test::registerTestVectorLowerings();
mlir::test::registerTestVectorReductionToSPIRVDotProd();
mlir::test::registerTestVulkanRunnerPipeline();
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 1b74d16736a47..5c7723136a149 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -1505,6 +1505,13 @@ td_library(
deps = [":OpBaseTdFiles"],
)
+td_library(
+ name = "InferUniformityOpInterfaceTdFiles",
+ srcs = ["include/mlir/Interfaces/InferUniformityOpInterface.td"],
+ includes = ["include"],
+ deps = [":OpBaseTdFiles"],
+)
+
td_library(
name = "LoopLikeInterfaceTdFiles",
srcs = ["include/mlir/Interfaces/LoopLikeInterface.td"],
@@ -1613,6 +1620,7 @@ td_library(
":ArithOpsTdFiles",
":FuncTdFiles",
":InferIntDivisibilityOpInterfaceTdFiles",
+ ":InferUniformityOpInterfaceTdFiles",
":LoopLikeInterfaceTdFiles",
":OpBaseTdFiles",
":SideEffectInterfacesTdFiles",
@@ -2830,6 +2838,7 @@ td_library(
":ControlFlowInterfacesTdFiles",
":DestinationStyleOpInterfaceTdFiles",
":InferTypeOpInterfaceTdFiles",
+ ":InferUniformityOpInterfaceTdFiles",
":LoopLikeInterfaceTdFiles",
":MemorySlotInterfacesTdFiles",
":ParallelCombiningOpInterfaceTdFiles",
@@ -4160,6 +4169,7 @@ cc_library(
":InferIntDivisibilityOpInterface",
":InferIntRangeCommon",
":InferIntRangeInterface",
+ ":InferUniformityOpInterface",
":InliningUtils",
":LoopLikeInterface",
":MemRefDialect",
@@ -4552,6 +4562,7 @@ cc_library(
":FunctionInterfaces",
":IR",
":InferTypeOpInterface",
+ ":InferUniformityOpInterface",
":InliningUtils",
":LoopLikeInterface",
":MemRefDialect",
@@ -5656,6 +5667,7 @@ td_library(
":DataLayoutInterfacesTdFiles",
":FunctionInterfacesTdFiles",
":InferIntRangeInterfaceTdFiles",
+ ":InferUniformityOpInterfaceTdFiles",
":LLVMOpsTdFiles",
":OpBaseTdFiles",
":SCFTdFiles",
@@ -5749,6 +5761,7 @@ cc_library(
":IR",
":InferIntRangeInterface",
":InferTypeOpInterface",
+ ":InferUniformityOpInterface",
":InliningUtils",
":MathDialect",
":MemRefDialect",
@@ -8064,6 +8077,29 @@ cc_library(
],
)
+gentbl_cc_library(
+ name = "InferUniformityOpInterfaceIncGen",
+ tbl_outs = {
+ "include/mlir/Interfaces/InferUniformityOpInterface.h.inc": ["-gen-op-interface-decls"],
+ "include/mlir/Interfaces/InferUniformityOpInterface.cpp.inc": ["-gen-op-interface-defs"],
+ },
+ tblgen = ":mlir-tblgen",
+ td_file = "include/mlir/Interfaces/InferUniformityOpInterface.td",
+ deps = [":InferUniformityOpInterfaceTdFiles"],
+)
+
+cc_library(
+ name = "InferUniformityOpInterface",
+ srcs = ["lib/Interfaces/InferUniformityOpInterface.cpp"],
+ hdrs = ["include/mlir/Interfaces/InferUniformityOpInterface.h"],
+ includes = ["include"],
+ deps = [
+ ":IR",
+ ":InferUniformityOpInterfaceIncGen",
+ "//llvm:Support",
+ ],
+)
+
td_library(
name = "DataLayoutInterfacesTdFiles",
srcs = ["include/mlir/Interfaces/DataLayoutInterfaces.td"],
@@ -9117,6 +9153,7 @@ cc_library(
":InferIntDivisibilityOpInterface",
":InferIntRangeInterface",
":InferStridedMetadataInterface",
+ ":InferUniformityOpInterface",
":LoopLikeInterface",
":Pass",
":SideEffectInterfaces",
More information about the Mlir-commits
mailing list