[Mlir-commits] [mlir] [mlir][gpu] Support `minimumnumf` and `maximumnumf` in GPU dialect (PR #226903)
Artem Gindinson
llvmlistbot at llvm.org
Tue Sep 29 04:08:44 PDT 2026
https://github.com/AGindinson updated https://github.com/llvm/llvm-project/pull/226903
>From dc3a2361aa0e7597531e5a347e3c56a74cec7e81 Mon Sep 17 00:00:00 2001
From: Artem Gindinson <gindinson at roofline.ai>
Date: Fri, 25 Sep 2026 15:24:44 +0000
Subject: [PATCH 1/2] [mlir][gpu] Support `minimumnumf` and `maximumnumf` in
GPU dialect
Follow up on #221658. Add mirroring GPU reduction kinds and lower
through appropriate arith ops.
Assisted-by: Codex
---
mlir/include/mlir/Dialect/GPU/IR/GPUOps.td | 12 +++++--
.../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp | 2 ++
mlir/lib/Dialect/GPU/IR/GPUDialect.cpp | 6 ++--
mlir/lib/Dialect/GPU/Utils/Utils.cpp | 2 ++
.../Dialect/GPU/all-reduce-minmaxnumf.mlir | 21 ++++++++++++
mlir/test/Dialect/GPU/invalid.mlir | 32 +++++++++++++++++++
6 files changed, 69 insertions(+), 6 deletions(-)
create mode 100644 mlir/test/Dialect/GPU/all-reduce-minmaxnumf.mlir
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 2aa5219e90404..1b6c4ec26ab68 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -1151,6 +1151,10 @@ def GPU_AllReduceOpXor : I32EnumAttrCase<"XOR", 10, "xor">;
def GPU_AllReduceOpMinimumF : I32EnumAttrCase<"MINIMUMF", 11, "minimumf">;
// Follows the `arith.maximumf` semantics.
def GPU_AllReduceOpMaximumF : I32EnumAttrCase<"MAXIMUMF", 12, "maximumf">;
+// Follows the `arith.minimumnumf` semantics.
+def GPU_AllReduceOpMinimumNumF : I32EnumAttrCase<"MINIMUMNUMF", 13, "minimumnumf">;
+// Follows the `arith.maximumnumf` semantics.
+def GPU_AllReduceOpMaximumNumF : I32EnumAttrCase<"MAXIMUMNUMF", 14, "maximumnumf">;
def GPU_AllReduceOperation : I32EnumAttr<"AllReduceOperation",
"built-in reduction operations supported by gpu.allreduce.",
@@ -1167,7 +1171,9 @@ def GPU_AllReduceOperation : I32EnumAttr<"AllReduceOperation",
GPU_AllReduceOpOr,
GPU_AllReduceOpXor,
GPU_AllReduceOpMinimumF,
- GPU_AllReduceOpMaximumF
+ GPU_AllReduceOpMaximumF,
+ GPU_AllReduceOpMinimumNumF,
+ GPU_AllReduceOpMaximumNumF
]>{
let genSpecializedAttr = 0;
let cppNamespace = "::mlir::gpu";
@@ -1202,7 +1208,7 @@ def GPU_AllReduceOp : GPU_Op<"all_reduce",
* Integer types: `add`, `mul`, `minui`, `minsi`, `maxui`, `maxsi`, `and`,
`or`, `xor`
* Floating point types: `add`, `mul`, `minnumf`, `maxnumf`, `minimumf`,
- `maximumf`
+ `maximumf`, `minimumnumf`, `maximumnumf`
If `uniform` flag is set either none or all work items of a workgroup
need to execute this op in convergence.
@@ -1259,7 +1265,7 @@ def GPU_SubgroupReduceOp : GPU_Op<"subgroup_reduce", [SameOperandsAndResultType,
* Integer types: `add`, `mul`, `minui`, `minsi`, `maxui`, `maxsi`, `and`,
`or`, `xor`
* Floating point types: `add`, `mul`, `minnumf`, `maxnumf`, `minimumf`,
- `maximumf`
+ `maximumf`, `minimumnumf`, `maximumnumf`
}];
let arguments = (ins
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 10de345eb6712..ea776000d8190 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -91,6 +91,8 @@ convertToNVVMReductionKind(gpu::AllReduceOperation mode) {
return NVVM::ReductionKind::XOR;
case gpu::AllReduceOperation::MINIMUMF:
case gpu::AllReduceOperation::MAXIMUMF:
+ case gpu::AllReduceOperation::MINIMUMNUMF:
+ case gpu::AllReduceOperation::MAXIMUMNUMF:
return std::nullopt;
}
return std::nullopt;
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index a39b43c9803eb..5c1a7004ebbc3 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -539,9 +539,9 @@ static LogicalResult verifyAttributions(Operation *op,
static LogicalResult verifyReduceOpAndType(gpu::AllReduceOperation opName,
Type resType) {
using Kind = gpu::AllReduceOperation;
- if (llvm::is_contained(
- {Kind::MINNUMF, Kind::MAXNUMF, Kind::MINIMUMF, Kind::MAXIMUMF},
- opName)) {
+ if (llvm::is_contained({Kind::MINNUMF, Kind::MAXNUMF, Kind::MINIMUMF,
+ Kind::MAXIMUMF, Kind::MINIMUMNUMF, Kind::MAXIMUMNUMF},
+ opName)) {
if (!isa<FloatType>(resType))
return failure();
}
diff --git a/mlir/lib/Dialect/GPU/Utils/Utils.cpp b/mlir/lib/Dialect/GPU/Utils/Utils.cpp
index 1f09875b3e273..8edd1ae904b6b 100644
--- a/mlir/lib/Dialect/GPU/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/Utils/Utils.cpp
@@ -34,6 +34,8 @@ vector::CombiningKind convertReductionKind(gpu::AllReduceOperation mode) {
MAP_CASE(XOR);
MAP_CASE(MINIMUMF);
MAP_CASE(MAXIMUMF);
+ MAP_CASE(MINIMUMNUMF);
+ MAP_CASE(MAXIMUMNUMF);
#undef MAP_CASE
}
diff --git a/mlir/test/Dialect/GPU/all-reduce-minmaxnumf.mlir b/mlir/test/Dialect/GPU/all-reduce-minmaxnumf.mlir
new file mode 100644
index 0000000000000..7ee88d38ffd4f
--- /dev/null
+++ b/mlir/test/Dialect/GPU/all-reduce-minmaxnumf.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt --convert-gpu-to-nvvm %s | FileCheck %s
+
+gpu.module @kernels {
+ // CHECK-LABEL: llvm.func @minimum_number
+ // CHECK: llvm.intr.minimumnum
+ // CHECK: llvm.return
+ gpu.func @minimum_number(%arg0 : f32, %out : memref<f32>) kernel {
+ %result = gpu.all_reduce minimumnumf %arg0 uniform {} : (f32) -> f32
+ memref.store %result, %out[] : memref<f32>
+ gpu.return
+ }
+
+ // CHECK-LABEL: llvm.func @maximum_number
+ // CHECK: llvm.intr.maximumnum
+ // CHECK: llvm.return
+ gpu.func @maximum_number(%arg0 : f32, %out : memref<f32>) kernel {
+ %result = gpu.all_reduce maximumnumf %arg0 uniform {} : (f32) -> f32
+ memref.store %result, %out[] : memref<f32>
+ gpu.return
+ }
+}
diff --git a/mlir/test/Dialect/GPU/invalid.mlir b/mlir/test/Dialect/GPU/invalid.mlir
index 1b6d3a41d2bd0..7d352506209ff 100644
--- a/mlir/test/Dialect/GPU/invalid.mlir
+++ b/mlir/test/Dialect/GPU/invalid.mlir
@@ -473,6 +473,22 @@ func.func @reduce_invalid_op_type_minimumf(%arg0 : i32) {
// -----
+func.func @reduce_invalid_op_type_minimumnumf(%arg0 : i32) {
+ // expected-error at +1 {{`minimumnumf` reduction operation is not compatible with type 'i32'}}
+ %res = gpu.all_reduce minimumnumf %arg0 {} : (i32) -> (i32)
+ return
+}
+
+// -----
+
+func.func @reduce_invalid_op_type_maximumnumf(%arg0 : i32) {
+ // expected-error at +1 {{`maximumnumf` reduction operation is not compatible with type 'i32'}}
+ %res = gpu.all_reduce maximumnumf %arg0 {} : (i32) -> (i32)
+ return
+}
+
+// -----
+
func.func @reduce_invalid_op_type_maximumf(%arg0 : i32) {
// expected-error at +1 {{`maximumf` reduction operation is not compatible with type 'i32'}}
%res = gpu.all_reduce maximumf %arg0 {} : (i32) -> (i32)
@@ -546,6 +562,22 @@ func.func @subgroup_reduce_invalid_op_type_maxnumf(%arg0 : i32) {
// -----
+func.func @subgroup_reduce_invalid_op_type_minimumnumf(%arg0 : i32) {
+ // expected-error at +1 {{`minimumnumf` reduction operation is not compatible with type 'i32'}}
+ %res = gpu.subgroup_reduce minimumnumf %arg0 : (i32) -> (i32)
+ return
+}
+
+// -----
+
+func.func @subgroup_reduce_invalid_op_type_maximumnumf(%arg0 : vector<2xi32>) {
+ // expected-error at +1 {{`maximumnumf` reduction operation is not compatible with type 'vector<2xi32>'}}
+ %res = gpu.subgroup_reduce maximumnumf %arg0 : (vector<2xi32>) -> vector<2xi32>
+ return
+}
+
+// -----
+
func.func @reduce_incorrect_region_arguments(%arg0 : f32) {
// expected-error at +1 {{expected two region arguments}}
%res = gpu.all_reduce %arg0 {
>From b3019a94bcfb54d31af634e0f3ff95017794f2e6 Mon Sep 17 00:00:00 2001
From: Artem Gindinson <gindinson at roofline.ai>
Date: Tue, 29 Sep 2026 10:02:12 +0000
Subject: [PATCH 2/2] [fixup] Address feedback: align LIT format and expand
minnum/maxnum tests
---
mlir/test/Dialect/GPU/all-reduce-maxf.mlir | 182 -------
mlir/test/Dialect/GPU/all-reduce-min-max.mlir | 466 ++++++++++++++++++
.../GPU/all-reduce-minimum-maximum.mlir | 466 ++++++++++++++++++
.../Dialect/GPU/all-reduce-minmaxnumf.mlir | 21 -
4 files changed, 932 insertions(+), 203 deletions(-)
delete mode 100644 mlir/test/Dialect/GPU/all-reduce-maxf.mlir
create mode 100644 mlir/test/Dialect/GPU/all-reduce-min-max.mlir
create mode 100644 mlir/test/Dialect/GPU/all-reduce-minimum-maximum.mlir
delete mode 100644 mlir/test/Dialect/GPU/all-reduce-minmaxnumf.mlir
diff --git a/mlir/test/Dialect/GPU/all-reduce-maxf.mlir b/mlir/test/Dialect/GPU/all-reduce-maxf.mlir
deleted file mode 100644
index 9c32249b2230f..0000000000000
--- a/mlir/test/Dialect/GPU/all-reduce-maxf.mlir
+++ /dev/null
@@ -1,182 +0,0 @@
-// RUN: mlir-opt -test-gpu-rewrite %s | FileCheck %s
-
-// NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
-// CHECK: gpu.module @kernels {
-gpu.module @kernels {
-
- // CHECK-LABEL: gpu.func @kernel(
- // CHECK-SAME: [[VAL_0:%.*]]: f32) workgroup([[VAL_1:%.*]] : memref<32xf32, #gpu.address_space<workgroup>>) kernel {
- gpu.func @kernel(%arg0 : f32) kernel {
- // CHECK-DAG: [[VAL_2:%.*]] = arith.constant 31 : i32
- // CHECK-DAG: [[VAL_3:%.*]] = arith.constant 0 : i32
- // CHECK-DAG: [[VAL_4:%.*]] = arith.constant 0 : index
- // CHECK-DAG: [[VAL_5:%.*]] = arith.constant 32 : i32
- // CHECK-DAG: [[VAL_6:%.*]] = arith.constant 1 : i32
- // CHECK-DAG: [[VAL_7:%.*]] = arith.constant 2 : i32
- // CHECK-DAG: [[VAL_8:%.*]] = arith.constant 4 : i32
- // CHECK-DAG: [[VAL_9:%.*]] = arith.constant 8 : i32
- // CHECK-DAG: [[VAL_10:%.*]] = arith.constant 16 : i32
- // CHECK: [[VAL_11:%.*]] = gpu.block_dim x
- // CHECK: [[VAL_12:%.*]] = arith.index_cast [[VAL_11]] : index to i32
- // CHECK: [[VAL_13:%.*]] = gpu.block_dim y
- // CHECK: [[VAL_14:%.*]] = arith.index_cast [[VAL_13]] : index to i32
- // CHECK: [[VAL_15:%.*]] = gpu.block_dim z
- // CHECK: [[VAL_16:%.*]] = arith.index_cast [[VAL_15]] : index to i32
- // CHECK: [[VAL_17:%.*]] = gpu.thread_id x
- // CHECK: [[VAL_18:%.*]] = arith.index_cast [[VAL_17]] : index to i32
- // CHECK: [[VAL_19:%.*]] = gpu.thread_id y
- // CHECK: [[VAL_20:%.*]] = arith.index_cast [[VAL_19]] : index to i32
- // CHECK: [[VAL_21:%.*]] = gpu.thread_id z
- // CHECK: [[VAL_22:%.*]] = arith.index_cast [[VAL_21]] : index to i32
- // CHECK: [[VAL_23:%.*]] = arith.muli [[VAL_22]], [[VAL_14]] : i32
- // CHECK: [[VAL_24:%.*]] = arith.addi [[VAL_23]], [[VAL_20]] : i32
- // CHECK: [[VAL_25:%.*]] = arith.muli [[VAL_24]], [[VAL_12]] : i32
- // CHECK: [[VAL_26:%.*]] = arith.muli [[VAL_12]], [[VAL_14]] : i32
- // CHECK: [[VAL_27:%.*]] = arith.addi [[VAL_25]], [[VAL_18]] : i32
- // CHECK: [[VAL_28:%.*]] = arith.muli [[VAL_26]], [[VAL_16]] : i32
- // CHECK: [[VAL_29:%.*]] = arith.andi [[VAL_27]], [[VAL_2]] : i32
- // CHECK: [[VAL_30:%.*]] = arith.cmpi eq, [[VAL_29]], [[VAL_3]] : i32
- // CHECK: [[VAL_31:%.*]] = arith.subi [[VAL_27]], [[VAL_29]] : i32
- // CHECK: [[VAL_32:%.*]] = arith.subi [[VAL_28]], [[VAL_31]] : i32
- // CHECK: [[VAL_33:%.*]] = arith.cmpi slt, [[VAL_32]], [[VAL_5]] : i32
- // CHECK: cf.cond_br [[VAL_33]], ^bb1, ^bb17
- // CHECK: ^bb1:
- // CHECK: [[VAL_34:%.*]], [[VAL_35:%.*]] = gpu.shuffle xor [[VAL_0]], [[VAL_6]], [[VAL_32]] : f32
- // CHECK: cf.cond_br [[VAL_35]], ^bb2, ^bb3
- // CHECK: ^bb2:
- // CHECK: [[VAL_36:%.*]] = arith.maxnumf [[VAL_0]], [[VAL_34]] : f32
- // CHECK: cf.br ^bb4([[VAL_36]] : f32)
- // CHECK: ^bb3:
- // CHECK: cf.br ^bb4([[VAL_0]] : f32)
- // CHECK: ^bb4([[VAL_38:%.*]]: f32):
- // CHECK: [[VAL_39:%.*]], [[VAL_40:%.*]] = gpu.shuffle xor [[VAL_38]], [[VAL_7]], [[VAL_32]] : f32
- // CHECK: cf.cond_br [[VAL_40]], ^bb5, ^bb6
- // CHECK: ^bb5:
- // CHECK: [[VAL_41:%.*]] = arith.maxnumf [[VAL_38]], [[VAL_39]] : f32
- // CHECK: cf.br ^bb7([[VAL_41]] : f32)
- // CHECK: ^bb6:
- // CHECK: cf.br ^bb7([[VAL_38]] : f32)
- // CHECK: ^bb7([[VAL_43:%.*]]: f32):
- // CHECK: [[VAL_44:%.*]], [[VAL_45:%.*]] = gpu.shuffle xor [[VAL_43]], [[VAL_8]], [[VAL_32]] : f32
- // CHECK: cf.cond_br [[VAL_45]], ^bb8, ^bb9
- // CHECK: ^bb8:
- // CHECK: [[VAL_46:%.*]] = arith.maxnumf [[VAL_43]], [[VAL_44]] : f32
- // CHECK: cf.br ^bb10([[VAL_46]] : f32)
- // CHECK: ^bb9:
- // CHECK: cf.br ^bb10([[VAL_43]] : f32)
- // CHECK: ^bb10([[VAL_48:%.*]]: f32):
- // CHECK: [[VAL_49:%.*]], [[VAL_50:%.*]] = gpu.shuffle xor [[VAL_48]], [[VAL_9]], [[VAL_32]] : f32
- // CHECK: cf.cond_br [[VAL_50]], ^bb11, ^bb12
- // CHECK: ^bb11:
- // CHECK: [[VAL_51:%.*]] = arith.maxnumf [[VAL_48]], [[VAL_49]] : f32
- // CHECK: cf.br ^bb13([[VAL_51]] : f32)
- // CHECK: ^bb12:
- // CHECK: cf.br ^bb13([[VAL_48]] : f32)
- // CHECK: ^bb13([[VAL_53:%.*]]: f32):
- // CHECK: [[VAL_54:%.*]], [[VAL_55:%.*]] = gpu.shuffle xor [[VAL_53]], [[VAL_10]], [[VAL_32]] : f32
- // CHECK: cf.cond_br [[VAL_55]], ^bb14, ^bb15
- // CHECK: ^bb14:
- // CHECK: [[VAL_56:%.*]] = arith.maxnumf [[VAL_53]], [[VAL_54]] : f32
- // CHECK: cf.br ^bb16([[VAL_56]] : f32)
- // CHECK: ^bb15:
- // CHECK: cf.br ^bb16([[VAL_53]] : f32)
- // CHECK: ^bb16([[VAL_58:%.*]]: f32):
- // CHECK: cf.br ^bb18([[VAL_58]] : f32)
- // CHECK: ^bb17:
- // CHECK: [[VAL_59:%.*]], [[VAL_60:%.*]] = gpu.shuffle xor [[VAL_0]], [[VAL_6]], [[VAL_5]] : f32
- // CHECK: [[VAL_62:%.*]] = arith.maxnumf [[VAL_0]], [[VAL_59]] : f32
- // CHECK: [[VAL_63:%.*]], [[VAL_64:%.*]] = gpu.shuffle xor [[VAL_62]], [[VAL_7]], [[VAL_5]] : f32
- // CHECK: [[VAL_66:%.*]] = arith.maxnumf [[VAL_62]], [[VAL_63]] : f32
- // CHECK: [[VAL_67:%.*]], [[VAL_68:%.*]] = gpu.shuffle xor [[VAL_66]], [[VAL_8]], [[VAL_5]] : f32
- // CHECK: [[VAL_70:%.*]] = arith.maxnumf [[VAL_66]], [[VAL_67]] : f32
- // CHECK: [[VAL_71:%.*]], [[VAL_72:%.*]] = gpu.shuffle xor [[VAL_70]], [[VAL_9]], [[VAL_5]] : f32
- // CHECK: [[VAL_74:%.*]] = arith.maxnumf [[VAL_70]], [[VAL_71]] : f32
- // CHECK: [[VAL_75:%.*]], [[VAL_76:%.*]] = gpu.shuffle xor [[VAL_74]], [[VAL_10]], [[VAL_5]] : f32
- // CHECK: [[VAL_78:%.*]] = arith.maxnumf [[VAL_74]], [[VAL_75]] : f32
- // CHECK: cf.br ^bb18([[VAL_78]] : f32)
- // CHECK: ^bb18([[VAL_79:%.*]]: f32):
- // CHECK: cf.cond_br [[VAL_30]], ^bb19, ^bb20
- // CHECK: ^bb19:
- // CHECK: [[VAL_80:%.*]] = arith.divsi [[VAL_27]], [[VAL_5]] : i32
- // CHECK: [[VAL_81:%.*]] = arith.index_cast [[VAL_80]] : i32 to index
- // CHECK: store [[VAL_79]], [[VAL_1]]{{\[}}[[VAL_81]]] : memref<32xf32, #gpu.address_space<workgroup>>
- // CHECK: cf.br ^bb21
- // CHECK: ^bb20:
- // CHECK: cf.br ^bb21
- // CHECK: ^bb21:
- // CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
- // CHECK: [[VAL_82:%.*]] = arith.addi [[VAL_28]], [[VAL_2]] : i32
- // CHECK: [[VAL_83:%.*]] = arith.divsi [[VAL_82]], [[VAL_5]] : i32
- // CHECK: [[VAL_84:%.*]] = arith.cmpi slt, [[VAL_27]], [[VAL_83]] : i32
- // CHECK: cf.cond_br [[VAL_84]], ^bb22, ^bb41
- // CHECK: ^bb22:
- // CHECK: [[VAL_85:%.*]] = arith.index_cast [[VAL_27]] : i32 to index
- // CHECK: [[VAL_86:%.*]] = memref.load [[VAL_1]]{{\[}}[[VAL_85]]] : memref<32xf32, #gpu.address_space<workgroup>>
- // CHECK: [[VAL_87:%.*]] = arith.cmpi slt, [[VAL_83]], [[VAL_5]] : i32
- // CHECK: cf.cond_br [[VAL_87]], ^bb23, ^bb39
- // CHECK: ^bb23:
- // CHECK: [[VAL_88:%.*]], [[VAL_89:%.*]] = gpu.shuffle xor [[VAL_86]], [[VAL_6]], [[VAL_83]] : f32
- // CHECK: cf.cond_br [[VAL_89]], ^bb24, ^bb25
- // CHECK: ^bb24:
- // CHECK: [[VAL_91:%.*]] = arith.maxnumf [[VAL_86]], [[VAL_88]] : f32
- // CHECK: cf.br ^bb26([[VAL_91]] : f32)
- // CHECK: ^bb25:
- // CHECK: cf.br ^bb26([[VAL_86]] : f32)
- // CHECK: ^bb26([[VAL_92:%.*]]: f32):
- // CHECK: [[VAL_93:%.*]], [[VAL_94:%.*]] = gpu.shuffle xor [[VAL_92]], [[VAL_7]], [[VAL_83]] : f32
- // CHECK: cf.cond_br [[VAL_94]], ^bb27, ^bb28
- // CHECK: ^bb27:
- // CHECK: [[VAL_96:%.*]] = arith.maxnumf [[VAL_92]], [[VAL_93]] : f32
- // CHECK: cf.br ^bb29([[VAL_96]] : f32)
- // CHECK: ^bb28:
- // CHECK: cf.br ^bb29([[VAL_92]] : f32)
- // CHECK: ^bb29([[VAL_97:%.*]]: f32):
- // CHECK: [[VAL_98:%.*]], [[VAL_99:%.*]] = gpu.shuffle xor [[VAL_97]], [[VAL_8]], [[VAL_83]] : f32
- // CHECK: cf.cond_br [[VAL_99]], ^bb30, ^bb31
- // CHECK: ^bb30:
- // CHECK: [[VAL_101:%.*]] = arith.maxnumf [[VAL_97]], [[VAL_98]] : f32
- // CHECK: cf.br ^bb32([[VAL_101]] : f32)
- // CHECK: ^bb31:
- // CHECK: cf.br ^bb32([[VAL_97]] : f32)
- // CHECK: ^bb32([[VAL_102:%.*]]: f32):
- // CHECK: [[VAL_103:%.*]], [[VAL_104:%.*]] = gpu.shuffle xor [[VAL_102]], [[VAL_9]], [[VAL_83]] : f32
- // CHECK: cf.cond_br [[VAL_104]], ^bb33, ^bb34
- // CHECK: ^bb33:
- // CHECK: [[VAL_106:%.*]] = arith.maxnumf [[VAL_102]], [[VAL_103]] : f32
- // CHECK: cf.br ^bb35([[VAL_106]] : f32)
- // CHECK: ^bb34:
- // CHECK: cf.br ^bb35([[VAL_102]] : f32)
- // CHECK: ^bb35([[VAL_107:%.*]]: f32):
- // CHECK: [[VAL_108:%.*]], [[VAL_109:%.*]] = gpu.shuffle xor [[VAL_107]], [[VAL_10]], [[VAL_83]] : f32
- // CHECK: cf.cond_br [[VAL_109]], ^bb36, ^bb37
- // CHECK: ^bb36:
- // CHECK: [[VAL_111:%.*]] = arith.maxnumf [[VAL_107]], [[VAL_108]] : f32
- // CHECK: cf.br ^bb38([[VAL_111]] : f32)
- // CHECK: ^bb37:
- // CHECK: cf.br ^bb38([[VAL_107]] : f32)
- // CHECK: ^bb38([[VAL_112:%.*]]: f32):
- // CHECK: cf.br ^bb40([[VAL_112]] : f32)
- // CHECK: ^bb39:
- // CHECK: [[VAL_113:%.*]], [[VAL_114:%.*]] = gpu.shuffle xor [[VAL_86]], [[VAL_6]], [[VAL_5]] : f32
- // CHECK: [[VAL_116:%.*]] = arith.maxnumf [[VAL_86]], [[VAL_113]] : f32
- // CHECK: [[VAL_117:%.*]], [[VAL_118:%.*]] = gpu.shuffle xor [[VAL_116]], [[VAL_7]], [[VAL_5]] : f32
- // CHECK: [[VAL_120:%.*]] = arith.maxnumf [[VAL_116]], [[VAL_117]] : f32
- // CHECK: [[VAL_121:%.*]], [[VAL_122:%.*]] = gpu.shuffle xor [[VAL_120]], [[VAL_8]], [[VAL_5]] : f32
- // CHECK: [[VAL_124:%.*]] = arith.maxnumf [[VAL_120]], [[VAL_121]] : f32
- // CHECK: [[VAL_125:%.*]], [[VAL_126:%.*]] = gpu.shuffle xor [[VAL_124]], [[VAL_9]], [[VAL_5]] : f32
- // CHECK: [[VAL_128:%.*]] = arith.maxnumf [[VAL_124]], [[VAL_125]] : f32
- // CHECK: [[VAL_129:%.*]], [[VAL_130:%.*]] = gpu.shuffle xor [[VAL_128]], [[VAL_10]], [[VAL_5]] : f32
- // CHECK: [[VAL_132:%.*]] = arith.maxnumf [[VAL_128]], [[VAL_129]] : f32
- // CHECK: cf.br ^bb40([[VAL_132]] : f32)
- // CHECK: ^bb40([[VAL_133:%.*]]: f32):
- // CHECK: store [[VAL_133]], [[VAL_1]]{{\[}}[[VAL_4]]] : memref<32xf32, #gpu.address_space<workgroup>>
- // CHECK: cf.br ^bb42
- // CHECK: ^bb41:
- // CHECK: cf.br ^bb42
- // CHECK: ^bb42:
- // CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
- %sum = gpu.all_reduce maxnumf %arg0 uniform {} : (f32) -> (f32)
- gpu.return
- }
-
-}
diff --git a/mlir/test/Dialect/GPU/all-reduce-min-max.mlir b/mlir/test/Dialect/GPU/all-reduce-min-max.mlir
new file mode 100644
index 0000000000000..d919dba2e7a04
--- /dev/null
+++ b/mlir/test/Dialect/GPU/all-reduce-min-max.mlir
@@ -0,0 +1,466 @@
+// NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
+
+// RUN: mlir-opt -test-gpu-rewrite \
+// RUN: -test-gpu-subgroup-reduce-lowering=expand-to-shuffles %s | FileCheck %s
+
+gpu.module @kernels {
+// CHECK-LABEL: gpu.func @min_number(
+// CHECK-SAME: %[[ARG0:.*]]: f32,
+// CHECK-SAME: %[[ARG1:.*]]: memref<f32>) workgroup(
+// CHECK-SAME: %[[ARG2:.*]] : memref<32xf32, #gpu.address_space<workgroup>>) kernel {
+// CHECK: %[[CONSTANT_0:.*]] = arith.constant 0 : index
+// CHECK: %[[CONSTANT_1:.*]] = arith.constant 16 : i32
+// CHECK: %[[CONSTANT_2:.*]] = arith.constant 8 : i32
+// CHECK: %[[CONSTANT_3:.*]] = arith.constant 4 : i32
+// CHECK: %[[CONSTANT_4:.*]] = arith.constant 2 : i32
+// CHECK: %[[CONSTANT_5:.*]] = arith.constant 1 : i32
+// CHECK: %[[CONSTANT_6:.*]] = arith.constant 32 : i32
+// CHECK: %[[CONSTANT_7:.*]] = arith.constant 0 : i32
+// CHECK: %[[CONSTANT_8:.*]] = arith.constant 31 : i32
+// CHECK: %[[BLOCK_DIM_0:.*]] = gpu.block_dim x
+// CHECK: %[[INDEX_CAST_0:.*]] = arith.index_cast %[[BLOCK_DIM_0]] : index to i32
+// CHECK: %[[BLOCK_DIM_1:.*]] = gpu.block_dim y
+// CHECK: %[[INDEX_CAST_1:.*]] = arith.index_cast %[[BLOCK_DIM_1]] : index to i32
+// CHECK: %[[BLOCK_DIM_2:.*]] = gpu.block_dim z
+// CHECK: %[[INDEX_CAST_2:.*]] = arith.index_cast %[[BLOCK_DIM_2]] : index to i32
+// CHECK: %[[THREAD_ID_0:.*]] = gpu.thread_id x
+// CHECK: %[[INDEX_CAST_3:.*]] = arith.index_cast %[[THREAD_ID_0]] : index to i32
+// CHECK: %[[THREAD_ID_1:.*]] = gpu.thread_id y
+// CHECK: %[[INDEX_CAST_4:.*]] = arith.index_cast %[[THREAD_ID_1]] : index to i32
+// CHECK: %[[THREAD_ID_2:.*]] = gpu.thread_id z
+// CHECK: %[[INDEX_CAST_5:.*]] = arith.index_cast %[[THREAD_ID_2]] : index to i32
+// CHECK: %[[MULI_0:.*]] = arith.muli %[[INDEX_CAST_5]], %[[INDEX_CAST_1]] : i32
+// CHECK: %[[ADDI_0:.*]] = arith.addi %[[MULI_0]], %[[INDEX_CAST_4]] : i32
+// CHECK: %[[MULI_1:.*]] = arith.muli %[[ADDI_0]], %[[INDEX_CAST_0]] : i32
+// CHECK: %[[MULI_2:.*]] = arith.muli %[[INDEX_CAST_0]], %[[INDEX_CAST_1]] : i32
+// CHECK: %[[ADDI_1:.*]] = arith.addi %[[MULI_1]], %[[INDEX_CAST_3]] : i32
+// CHECK: %[[MULI_3:.*]] = arith.muli %[[MULI_2]], %[[INDEX_CAST_2]] : i32
+// CHECK: %[[ANDI_0:.*]] = arith.andi %[[ADDI_1]], %[[CONSTANT_8]] : i32
+// CHECK: %[[CMPI_0:.*]] = arith.cmpi eq, %[[ANDI_0]], %[[CONSTANT_7]] : i32
+// CHECK: %[[SUBI_0:.*]] = arith.subi %[[ADDI_1]], %[[ANDI_0]] : i32
+// CHECK: %[[SUBI_1:.*]] = arith.subi %[[MULI_3]], %[[SUBI_0]] : i32
+// CHECK: %[[CMPI_1:.*]] = arith.cmpi slt, %[[SUBI_1]], %[[CONSTANT_6]] : i32
+// CHECK: cf.cond_br %[[CMPI_1]], ^bb1, ^bb17
+// CHECK: ^bb1:
+// CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_0]], ^bb2, ^bb3
+// CHECK: ^bb2:
+// CHECK: %[[MINNUMF_0:.*]] = arith.minnumf %[[ARG0]], %[[VAL_0]] : f32
+// CHECK: cf.br ^bb4(%[[MINNUMF_0]] : f32)
+// CHECK: ^bb3:
+// CHECK: cf.br ^bb4(%[[ARG0]] : f32)
+// CHECK: ^bb4(%[[VAL_1:.*]]: f32):
+// CHECK: %[[VAL_2:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[VAL_1]], %[[CONSTANT_4]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_1]], ^bb5, ^bb6
+// CHECK: ^bb5:
+// CHECK: %[[MINNUMF_1:.*]] = arith.minnumf %[[VAL_1]], %[[VAL_2]] : f32
+// CHECK: cf.br ^bb7(%[[MINNUMF_1]] : f32)
+// CHECK: ^bb6:
+// CHECK: cf.br ^bb7(%[[VAL_1]] : f32)
+// CHECK: ^bb7(%[[VAL_3:.*]]: f32):
+// CHECK: %[[VAL_4:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[VAL_3]], %[[CONSTANT_3]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_2]], ^bb8, ^bb9
+// CHECK: ^bb8:
+// CHECK: %[[MINNUMF_2:.*]] = arith.minnumf %[[VAL_3]], %[[VAL_4]] : f32
+// CHECK: cf.br ^bb10(%[[MINNUMF_2]] : f32)
+// CHECK: ^bb9:
+// CHECK: cf.br ^bb10(%[[VAL_3]] : f32)
+// CHECK: ^bb10(%[[VAL_5:.*]]: f32):
+// CHECK: %[[VAL_6:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[VAL_5]], %[[CONSTANT_2]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_3]], ^bb11, ^bb12
+// CHECK: ^bb11:
+// CHECK: %[[MINNUMF_3:.*]] = arith.minnumf %[[VAL_5]], %[[VAL_6]] : f32
+// CHECK: cf.br ^bb13(%[[MINNUMF_3]] : f32)
+// CHECK: ^bb12:
+// CHECK: cf.br ^bb13(%[[VAL_5]] : f32)
+// CHECK: ^bb13(%[[VAL_7:.*]]: f32):
+// CHECK: %[[VAL_8:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[VAL_7]], %[[CONSTANT_1]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_4]], ^bb14, ^bb15
+// CHECK: ^bb14:
+// CHECK: %[[MINNUMF_4:.*]] = arith.minnumf %[[VAL_7]], %[[VAL_8]] : f32
+// CHECK: cf.br ^bb16(%[[MINNUMF_4]] : f32)
+// CHECK: ^bb15:
+// CHECK: cf.br ^bb16(%[[VAL_7]] : f32)
+// CHECK: ^bb16(%[[VAL_9:.*]]: f32):
+// CHECK: cf.br ^bb18(%[[VAL_9]] : f32)
+// CHECK: ^bb17:
+// CHECK: %[[VAL_10:.*]], %[[SHUFFLE_5:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_5:.*]] = arith.minnumf %[[ARG0]], %[[VAL_10]] : f32
+// CHECK: %[[VAL_11:.*]], %[[SHUFFLE_6:.*]] = gpu.shuffle xor %[[MINNUMF_5]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_6:.*]] = arith.minnumf %[[MINNUMF_5]], %[[VAL_11]] : f32
+// CHECK: %[[VAL_12:.*]], %[[SHUFFLE_7:.*]] = gpu.shuffle xor %[[MINNUMF_6]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_7:.*]] = arith.minnumf %[[MINNUMF_6]], %[[VAL_12]] : f32
+// CHECK: %[[VAL_13:.*]], %[[SHUFFLE_8:.*]] = gpu.shuffle xor %[[MINNUMF_7]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_8:.*]] = arith.minnumf %[[MINNUMF_7]], %[[VAL_13]] : f32
+// CHECK: %[[VAL_14:.*]], %[[SHUFFLE_9:.*]] = gpu.shuffle xor %[[MINNUMF_8]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_9:.*]] = arith.minnumf %[[MINNUMF_8]], %[[VAL_14]] : f32
+// CHECK: cf.br ^bb18(%[[MINNUMF_9]] : f32)
+// CHECK: ^bb18(%[[VAL_15:.*]]: f32):
+// CHECK: cf.cond_br %[[CMPI_0]], ^bb19, ^bb20
+// CHECK: ^bb19:
+// CHECK: %[[DIVSI_0:.*]] = arith.divsi %[[ADDI_1]], %[[CONSTANT_6]] : i32
+// CHECK: %[[INDEX_CAST_6:.*]] = arith.index_cast %[[DIVSI_0]] : i32 to index
+// CHECK: memref.store %[[VAL_15]], %[[ARG2]]{{\[}}%[[INDEX_CAST_6]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: cf.br ^bb21
+// CHECK: ^bb20:
+// CHECK: cf.br ^bb21
+// CHECK: ^bb21:
+// CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
+// CHECK: %[[ADDI_2:.*]] = arith.addi %[[MULI_3]], %[[CONSTANT_8]] : i32
+// CHECK: %[[DIVSI_1:.*]] = arith.divsi %[[ADDI_2]], %[[CONSTANT_6]] : i32
+// CHECK: %[[CMPI_2:.*]] = arith.cmpi slt, %[[ADDI_1]], %[[DIVSI_1]] : i32
+// CHECK: cf.cond_br %[[CMPI_2]], ^bb22, ^bb41
+// CHECK: ^bb22:
+// CHECK: %[[INDEX_CAST_7:.*]] = arith.index_cast %[[ADDI_1]] : i32 to index
+// CHECK: %[[LOAD_0:.*]] = memref.load %[[ARG2]]{{\[}}%[[INDEX_CAST_7]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: %[[CMPI_3:.*]] = arith.cmpi slt, %[[DIVSI_1]], %[[CONSTANT_6]] : i32
+// CHECK: cf.cond_br %[[CMPI_3]], ^bb23, ^bb39
+// CHECK: ^bb23:
+// CHECK: %[[VAL_16:.*]], %[[SHUFFLE_10:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_10]], ^bb24, ^bb25
+// CHECK: ^bb24:
+// CHECK: %[[MINNUMF_10:.*]] = arith.minnumf %[[LOAD_0]], %[[VAL_16]] : f32
+// CHECK: cf.br ^bb26(%[[MINNUMF_10]] : f32)
+// CHECK: ^bb25:
+// CHECK: cf.br ^bb26(%[[LOAD_0]] : f32)
+// CHECK: ^bb26(%[[VAL_17:.*]]: f32):
+// CHECK: %[[VAL_18:.*]], %[[SHUFFLE_11:.*]] = gpu.shuffle xor %[[VAL_17]], %[[CONSTANT_4]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_11]], ^bb27, ^bb28
+// CHECK: ^bb27:
+// CHECK: %[[MINNUMF_11:.*]] = arith.minnumf %[[VAL_17]], %[[VAL_18]] : f32
+// CHECK: cf.br ^bb29(%[[MINNUMF_11]] : f32)
+// CHECK: ^bb28:
+// CHECK: cf.br ^bb29(%[[VAL_17]] : f32)
+// CHECK: ^bb29(%[[VAL_19:.*]]: f32):
+// CHECK: %[[VAL_20:.*]], %[[SHUFFLE_12:.*]] = gpu.shuffle xor %[[VAL_19]], %[[CONSTANT_3]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_12]], ^bb30, ^bb31
+// CHECK: ^bb30:
+// CHECK: %[[MINNUMF_12:.*]] = arith.minnumf %[[VAL_19]], %[[VAL_20]] : f32
+// CHECK: cf.br ^bb32(%[[MINNUMF_12]] : f32)
+// CHECK: ^bb31:
+// CHECK: cf.br ^bb32(%[[VAL_19]] : f32)
+// CHECK: ^bb32(%[[VAL_21:.*]]: f32):
+// CHECK: %[[VAL_22:.*]], %[[SHUFFLE_13:.*]] = gpu.shuffle xor %[[VAL_21]], %[[CONSTANT_2]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_13]], ^bb33, ^bb34
+// CHECK: ^bb33:
+// CHECK: %[[MINNUMF_13:.*]] = arith.minnumf %[[VAL_21]], %[[VAL_22]] : f32
+// CHECK: cf.br ^bb35(%[[MINNUMF_13]] : f32)
+// CHECK: ^bb34:
+// CHECK: cf.br ^bb35(%[[VAL_21]] : f32)
+// CHECK: ^bb35(%[[VAL_23:.*]]: f32):
+// CHECK: %[[VAL_24:.*]], %[[SHUFFLE_14:.*]] = gpu.shuffle xor %[[VAL_23]], %[[CONSTANT_1]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_14]], ^bb36, ^bb37
+// CHECK: ^bb36:
+// CHECK: %[[MINNUMF_14:.*]] = arith.minnumf %[[VAL_23]], %[[VAL_24]] : f32
+// CHECK: cf.br ^bb38(%[[MINNUMF_14]] : f32)
+// CHECK: ^bb37:
+// CHECK: cf.br ^bb38(%[[VAL_23]] : f32)
+// CHECK: ^bb38(%[[VAL_25:.*]]: f32):
+// CHECK: cf.br ^bb40(%[[VAL_25]] : f32)
+// CHECK: ^bb39:
+// CHECK: %[[VAL_26:.*]], %[[SHUFFLE_15:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_15:.*]] = arith.minnumf %[[LOAD_0]], %[[VAL_26]] : f32
+// CHECK: %[[VAL_27:.*]], %[[SHUFFLE_16:.*]] = gpu.shuffle xor %[[MINNUMF_15]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_16:.*]] = arith.minnumf %[[MINNUMF_15]], %[[VAL_27]] : f32
+// CHECK: %[[VAL_28:.*]], %[[SHUFFLE_17:.*]] = gpu.shuffle xor %[[MINNUMF_16]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_17:.*]] = arith.minnumf %[[MINNUMF_16]], %[[VAL_28]] : f32
+// CHECK: %[[VAL_29:.*]], %[[SHUFFLE_18:.*]] = gpu.shuffle xor %[[MINNUMF_17]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_18:.*]] = arith.minnumf %[[MINNUMF_17]], %[[VAL_29]] : f32
+// CHECK: %[[VAL_30:.*]], %[[SHUFFLE_19:.*]] = gpu.shuffle xor %[[MINNUMF_18]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINNUMF_19:.*]] = arith.minnumf %[[MINNUMF_18]], %[[VAL_30]] : f32
+// CHECK: cf.br ^bb40(%[[MINNUMF_19]] : f32)
+// CHECK: ^bb40(%[[VAL_31:.*]]: f32):
+// CHECK: memref.store %[[VAL_31]], %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: cf.br ^bb42
+// CHECK: ^bb41:
+// CHECK: cf.br ^bb42
+// CHECK: ^bb42:
+// CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
+// CHECK: %[[LOAD_1:.*]] = memref.load %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: memref.store %[[LOAD_1]], %[[ARG1]][] : memref<f32>
+// CHECK: gpu.return
+// CHECK: }
+ gpu.func @min_number(%arg0 : f32, %out : memref<f32>) kernel {
+ %result = gpu.all_reduce minnumf %arg0 uniform {} : (f32) -> f32
+ memref.store %result, %out[] : memref<f32>
+ gpu.return
+ }
+
+// CHECK-LABEL: gpu.func @max_number(
+// CHECK-SAME: %[[ARG0:.*]]: f32,
+// CHECK-SAME: %[[ARG1:.*]]: memref<f32>) workgroup(
+// CHECK-SAME: %[[ARG2:.*]] : memref<32xf32, #gpu.address_space<workgroup>>) kernel {
+// CHECK: %[[CONSTANT_0:.*]] = arith.constant 0 : index
+// CHECK: %[[CONSTANT_1:.*]] = arith.constant 16 : i32
+// CHECK: %[[CONSTANT_2:.*]] = arith.constant 8 : i32
+// CHECK: %[[CONSTANT_3:.*]] = arith.constant 4 : i32
+// CHECK: %[[CONSTANT_4:.*]] = arith.constant 2 : i32
+// CHECK: %[[CONSTANT_5:.*]] = arith.constant 1 : i32
+// CHECK: %[[CONSTANT_6:.*]] = arith.constant 32 : i32
+// CHECK: %[[CONSTANT_7:.*]] = arith.constant 0 : i32
+// CHECK: %[[CONSTANT_8:.*]] = arith.constant 31 : i32
+// CHECK: %[[BLOCK_DIM_0:.*]] = gpu.block_dim x
+// CHECK: %[[INDEX_CAST_0:.*]] = arith.index_cast %[[BLOCK_DIM_0]] : index to i32
+// CHECK: %[[BLOCK_DIM_1:.*]] = gpu.block_dim y
+// CHECK: %[[INDEX_CAST_1:.*]] = arith.index_cast %[[BLOCK_DIM_1]] : index to i32
+// CHECK: %[[BLOCK_DIM_2:.*]] = gpu.block_dim z
+// CHECK: %[[INDEX_CAST_2:.*]] = arith.index_cast %[[BLOCK_DIM_2]] : index to i32
+// CHECK: %[[THREAD_ID_0:.*]] = gpu.thread_id x
+// CHECK: %[[INDEX_CAST_3:.*]] = arith.index_cast %[[THREAD_ID_0]] : index to i32
+// CHECK: %[[THREAD_ID_1:.*]] = gpu.thread_id y
+// CHECK: %[[INDEX_CAST_4:.*]] = arith.index_cast %[[THREAD_ID_1]] : index to i32
+// CHECK: %[[THREAD_ID_2:.*]] = gpu.thread_id z
+// CHECK: %[[INDEX_CAST_5:.*]] = arith.index_cast %[[THREAD_ID_2]] : index to i32
+// CHECK: %[[MULI_0:.*]] = arith.muli %[[INDEX_CAST_5]], %[[INDEX_CAST_1]] : i32
+// CHECK: %[[ADDI_0:.*]] = arith.addi %[[MULI_0]], %[[INDEX_CAST_4]] : i32
+// CHECK: %[[MULI_1:.*]] = arith.muli %[[ADDI_0]], %[[INDEX_CAST_0]] : i32
+// CHECK: %[[MULI_2:.*]] = arith.muli %[[INDEX_CAST_0]], %[[INDEX_CAST_1]] : i32
+// CHECK: %[[ADDI_1:.*]] = arith.addi %[[MULI_1]], %[[INDEX_CAST_3]] : i32
+// CHECK: %[[MULI_3:.*]] = arith.muli %[[MULI_2]], %[[INDEX_CAST_2]] : i32
+// CHECK: %[[ANDI_0:.*]] = arith.andi %[[ADDI_1]], %[[CONSTANT_8]] : i32
+// CHECK: %[[CMPI_0:.*]] = arith.cmpi eq, %[[ANDI_0]], %[[CONSTANT_7]] : i32
+// CHECK: %[[SUBI_0:.*]] = arith.subi %[[ADDI_1]], %[[ANDI_0]] : i32
+// CHECK: %[[SUBI_1:.*]] = arith.subi %[[MULI_3]], %[[SUBI_0]] : i32
+// CHECK: %[[CMPI_1:.*]] = arith.cmpi slt, %[[SUBI_1]], %[[CONSTANT_6]] : i32
+// CHECK: cf.cond_br %[[CMPI_1]], ^bb1, ^bb17
+// CHECK: ^bb1:
+// CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_0]], ^bb2, ^bb3
+// CHECK: ^bb2:
+// CHECK: %[[MAXNUMF_0:.*]] = arith.maxnumf %[[ARG0]], %[[VAL_0]] : f32
+// CHECK: cf.br ^bb4(%[[MAXNUMF_0]] : f32)
+// CHECK: ^bb3:
+// CHECK: cf.br ^bb4(%[[ARG0]] : f32)
+// CHECK: ^bb4(%[[VAL_1:.*]]: f32):
+// CHECK: %[[VAL_2:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[VAL_1]], %[[CONSTANT_4]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_1]], ^bb5, ^bb6
+// CHECK: ^bb5:
+// CHECK: %[[MAXNUMF_1:.*]] = arith.maxnumf %[[VAL_1]], %[[VAL_2]] : f32
+// CHECK: cf.br ^bb7(%[[MAXNUMF_1]] : f32)
+// CHECK: ^bb6:
+// CHECK: cf.br ^bb7(%[[VAL_1]] : f32)
+// CHECK: ^bb7(%[[VAL_3:.*]]: f32):
+// CHECK: %[[VAL_4:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[VAL_3]], %[[CONSTANT_3]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_2]], ^bb8, ^bb9
+// CHECK: ^bb8:
+// CHECK: %[[MAXNUMF_2:.*]] = arith.maxnumf %[[VAL_3]], %[[VAL_4]] : f32
+// CHECK: cf.br ^bb10(%[[MAXNUMF_2]] : f32)
+// CHECK: ^bb9:
+// CHECK: cf.br ^bb10(%[[VAL_3]] : f32)
+// CHECK: ^bb10(%[[VAL_5:.*]]: f32):
+// CHECK: %[[VAL_6:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[VAL_5]], %[[CONSTANT_2]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_3]], ^bb11, ^bb12
+// CHECK: ^bb11:
+// CHECK: %[[MAXNUMF_3:.*]] = arith.maxnumf %[[VAL_5]], %[[VAL_6]] : f32
+// CHECK: cf.br ^bb13(%[[MAXNUMF_3]] : f32)
+// CHECK: ^bb12:
+// CHECK: cf.br ^bb13(%[[VAL_5]] : f32)
+// CHECK: ^bb13(%[[VAL_7:.*]]: f32):
+// CHECK: %[[VAL_8:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[VAL_7]], %[[CONSTANT_1]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_4]], ^bb14, ^bb15
+// CHECK: ^bb14:
+// CHECK: %[[MAXNUMF_4:.*]] = arith.maxnumf %[[VAL_7]], %[[VAL_8]] : f32
+// CHECK: cf.br ^bb16(%[[MAXNUMF_4]] : f32)
+// CHECK: ^bb15:
+// CHECK: cf.br ^bb16(%[[VAL_7]] : f32)
+// CHECK: ^bb16(%[[VAL_9:.*]]: f32):
+// CHECK: cf.br ^bb18(%[[VAL_9]] : f32)
+// CHECK: ^bb17:
+// CHECK: %[[VAL_10:.*]], %[[SHUFFLE_5:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_5:.*]] = arith.maxnumf %[[ARG0]], %[[VAL_10]] : f32
+// CHECK: %[[VAL_11:.*]], %[[SHUFFLE_6:.*]] = gpu.shuffle xor %[[MAXNUMF_5]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_6:.*]] = arith.maxnumf %[[MAXNUMF_5]], %[[VAL_11]] : f32
+// CHECK: %[[VAL_12:.*]], %[[SHUFFLE_7:.*]] = gpu.shuffle xor %[[MAXNUMF_6]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_7:.*]] = arith.maxnumf %[[MAXNUMF_6]], %[[VAL_12]] : f32
+// CHECK: %[[VAL_13:.*]], %[[SHUFFLE_8:.*]] = gpu.shuffle xor %[[MAXNUMF_7]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_8:.*]] = arith.maxnumf %[[MAXNUMF_7]], %[[VAL_13]] : f32
+// CHECK: %[[VAL_14:.*]], %[[SHUFFLE_9:.*]] = gpu.shuffle xor %[[MAXNUMF_8]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_9:.*]] = arith.maxnumf %[[MAXNUMF_8]], %[[VAL_14]] : f32
+// CHECK: cf.br ^bb18(%[[MAXNUMF_9]] : f32)
+// CHECK: ^bb18(%[[VAL_15:.*]]: f32):
+// CHECK: cf.cond_br %[[CMPI_0]], ^bb19, ^bb20
+// CHECK: ^bb19:
+// CHECK: %[[DIVSI_0:.*]] = arith.divsi %[[ADDI_1]], %[[CONSTANT_6]] : i32
+// CHECK: %[[INDEX_CAST_6:.*]] = arith.index_cast %[[DIVSI_0]] : i32 to index
+// CHECK: memref.store %[[VAL_15]], %[[ARG2]]{{\[}}%[[INDEX_CAST_6]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: cf.br ^bb21
+// CHECK: ^bb20:
+// CHECK: cf.br ^bb21
+// CHECK: ^bb21:
+// CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
+// CHECK: %[[ADDI_2:.*]] = arith.addi %[[MULI_3]], %[[CONSTANT_8]] : i32
+// CHECK: %[[DIVSI_1:.*]] = arith.divsi %[[ADDI_2]], %[[CONSTANT_6]] : i32
+// CHECK: %[[CMPI_2:.*]] = arith.cmpi slt, %[[ADDI_1]], %[[DIVSI_1]] : i32
+// CHECK: cf.cond_br %[[CMPI_2]], ^bb22, ^bb41
+// CHECK: ^bb22:
+// CHECK: %[[INDEX_CAST_7:.*]] = arith.index_cast %[[ADDI_1]] : i32 to index
+// CHECK: %[[LOAD_0:.*]] = memref.load %[[ARG2]]{{\[}}%[[INDEX_CAST_7]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: %[[CMPI_3:.*]] = arith.cmpi slt, %[[DIVSI_1]], %[[CONSTANT_6]] : i32
+// CHECK: cf.cond_br %[[CMPI_3]], ^bb23, ^bb39
+// CHECK: ^bb23:
+// CHECK: %[[VAL_16:.*]], %[[SHUFFLE_10:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_10]], ^bb24, ^bb25
+// CHECK: ^bb24:
+// CHECK: %[[MAXNUMF_10:.*]] = arith.maxnumf %[[LOAD_0]], %[[VAL_16]] : f32
+// CHECK: cf.br ^bb26(%[[MAXNUMF_10]] : f32)
+// CHECK: ^bb25:
+// CHECK: cf.br ^bb26(%[[LOAD_0]] : f32)
+// CHECK: ^bb26(%[[VAL_17:.*]]: f32):
+// CHECK: %[[VAL_18:.*]], %[[SHUFFLE_11:.*]] = gpu.shuffle xor %[[VAL_17]], %[[CONSTANT_4]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_11]], ^bb27, ^bb28
+// CHECK: ^bb27:
+// CHECK: %[[MAXNUMF_11:.*]] = arith.maxnumf %[[VAL_17]], %[[VAL_18]] : f32
+// CHECK: cf.br ^bb29(%[[MAXNUMF_11]] : f32)
+// CHECK: ^bb28:
+// CHECK: cf.br ^bb29(%[[VAL_17]] : f32)
+// CHECK: ^bb29(%[[VAL_19:.*]]: f32):
+// CHECK: %[[VAL_20:.*]], %[[SHUFFLE_12:.*]] = gpu.shuffle xor %[[VAL_19]], %[[CONSTANT_3]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_12]], ^bb30, ^bb31
+// CHECK: ^bb30:
+// CHECK: %[[MAXNUMF_12:.*]] = arith.maxnumf %[[VAL_19]], %[[VAL_20]] : f32
+// CHECK: cf.br ^bb32(%[[MAXNUMF_12]] : f32)
+// CHECK: ^bb31:
+// CHECK: cf.br ^bb32(%[[VAL_19]] : f32)
+// CHECK: ^bb32(%[[VAL_21:.*]]: f32):
+// CHECK: %[[VAL_22:.*]], %[[SHUFFLE_13:.*]] = gpu.shuffle xor %[[VAL_21]], %[[CONSTANT_2]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_13]], ^bb33, ^bb34
+// CHECK: ^bb33:
+// CHECK: %[[MAXNUMF_13:.*]] = arith.maxnumf %[[VAL_21]], %[[VAL_22]] : f32
+// CHECK: cf.br ^bb35(%[[MAXNUMF_13]] : f32)
+// CHECK: ^bb34:
+// CHECK: cf.br ^bb35(%[[VAL_21]] : f32)
+// CHECK: ^bb35(%[[VAL_23:.*]]: f32):
+// CHECK: %[[VAL_24:.*]], %[[SHUFFLE_14:.*]] = gpu.shuffle xor %[[VAL_23]], %[[CONSTANT_1]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_14]], ^bb36, ^bb37
+// CHECK: ^bb36:
+// CHECK: %[[MAXNUMF_14:.*]] = arith.maxnumf %[[VAL_23]], %[[VAL_24]] : f32
+// CHECK: cf.br ^bb38(%[[MAXNUMF_14]] : f32)
+// CHECK: ^bb37:
+// CHECK: cf.br ^bb38(%[[VAL_23]] : f32)
+// CHECK: ^bb38(%[[VAL_25:.*]]: f32):
+// CHECK: cf.br ^bb40(%[[VAL_25]] : f32)
+// CHECK: ^bb39:
+// CHECK: %[[VAL_26:.*]], %[[SHUFFLE_15:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_15:.*]] = arith.maxnumf %[[LOAD_0]], %[[VAL_26]] : f32
+// CHECK: %[[VAL_27:.*]], %[[SHUFFLE_16:.*]] = gpu.shuffle xor %[[MAXNUMF_15]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_16:.*]] = arith.maxnumf %[[MAXNUMF_15]], %[[VAL_27]] : f32
+// CHECK: %[[VAL_28:.*]], %[[SHUFFLE_17:.*]] = gpu.shuffle xor %[[MAXNUMF_16]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_17:.*]] = arith.maxnumf %[[MAXNUMF_16]], %[[VAL_28]] : f32
+// CHECK: %[[VAL_29:.*]], %[[SHUFFLE_18:.*]] = gpu.shuffle xor %[[MAXNUMF_17]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_18:.*]] = arith.maxnumf %[[MAXNUMF_17]], %[[VAL_29]] : f32
+// CHECK: %[[VAL_30:.*]], %[[SHUFFLE_19:.*]] = gpu.shuffle xor %[[MAXNUMF_18]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXNUMF_19:.*]] = arith.maxnumf %[[MAXNUMF_18]], %[[VAL_30]] : f32
+// CHECK: cf.br ^bb40(%[[MAXNUMF_19]] : f32)
+// CHECK: ^bb40(%[[VAL_31:.*]]: f32):
+// CHECK: memref.store %[[VAL_31]], %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: cf.br ^bb42
+// CHECK: ^bb41:
+// CHECK: cf.br ^bb42
+// CHECK: ^bb42:
+// CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
+// CHECK: %[[LOAD_1:.*]] = memref.load %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: memref.store %[[LOAD_1]], %[[ARG1]][] : memref<f32>
+// CHECK: gpu.return
+// CHECK: }
+ gpu.func @max_number(%arg0 : f32, %out : memref<f32>) kernel {
+ %result = gpu.all_reduce maxnumf %arg0 uniform {} : (f32) -> f32
+ memref.store %result, %out[] : memref<f32>
+ gpu.return
+ }
+
+// CHECK-LABEL: gpu.func @min_number_vector(
+// CHECK-SAME: %[[ARG0:.*]]: vector<2xf16>,
+// CHECK-SAME: %[[ARG1:.*]]: memref<vector<2xf16>>) kernel {
+// CHECK: %[[CONSTANT_0:.*]] = arith.constant 16 : i32
+// CHECK: %[[CONSTANT_1:.*]] = arith.constant 8 : i32
+// CHECK: %[[CONSTANT_2:.*]] = arith.constant 4 : i32
+// CHECK: %[[CONSTANT_3:.*]] = arith.constant 2 : i32
+// CHECK: %[[CONSTANT_4:.*]] = arith.constant 32 : i32
+// CHECK: %[[CONSTANT_5:.*]] = arith.constant 1 : i32
+// CHECK: %[[BITCAST_0:.*]] = vector.bitcast %[[ARG0]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_0:.*]] = vector.extract %[[BITCAST_0]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[EXTRACT_0]], %[[CONSTANT_5]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_0:.*]] = vector.broadcast %[[VAL_0]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_1:.*]] = vector.bitcast %[[BROADCAST_0]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINNUMF_0:.*]] = arith.minnumf %[[ARG0]], %[[BITCAST_1]] : vector<2xf16>
+// CHECK: %[[BITCAST_2:.*]] = vector.bitcast %[[MINNUMF_0]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_1:.*]] = vector.extract %[[BITCAST_2]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_1:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[EXTRACT_1]], %[[CONSTANT_3]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_1:.*]] = vector.broadcast %[[VAL_1]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_3:.*]] = vector.bitcast %[[BROADCAST_1]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINNUMF_1:.*]] = arith.minnumf %[[MINNUMF_0]], %[[BITCAST_3]] : vector<2xf16>
+// CHECK: %[[BITCAST_4:.*]] = vector.bitcast %[[MINNUMF_1]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_2:.*]] = vector.extract %[[BITCAST_4]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_2:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[EXTRACT_2]], %[[CONSTANT_2]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_2:.*]] = vector.broadcast %[[VAL_2]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_5:.*]] = vector.bitcast %[[BROADCAST_2]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINNUMF_2:.*]] = arith.minnumf %[[MINNUMF_1]], %[[BITCAST_5]] : vector<2xf16>
+// CHECK: %[[BITCAST_6:.*]] = vector.bitcast %[[MINNUMF_2]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_3:.*]] = vector.extract %[[BITCAST_6]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_3:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[EXTRACT_3]], %[[CONSTANT_1]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_3:.*]] = vector.broadcast %[[VAL_3]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_7:.*]] = vector.bitcast %[[BROADCAST_3]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINNUMF_3:.*]] = arith.minnumf %[[MINNUMF_2]], %[[BITCAST_7]] : vector<2xf16>
+// CHECK: %[[BITCAST_8:.*]] = vector.bitcast %[[MINNUMF_3]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_4:.*]] = vector.extract %[[BITCAST_8]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_4:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[EXTRACT_4]], %[[CONSTANT_0]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_4:.*]] = vector.broadcast %[[VAL_4]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_9:.*]] = vector.bitcast %[[BROADCAST_4]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINNUMF_4:.*]] = arith.minnumf %[[MINNUMF_3]], %[[BITCAST_9]] : vector<2xf16>
+// CHECK: memref.store %[[MINNUMF_4]], %[[ARG1]][] : memref<vector<2xf16>>
+// CHECK: gpu.return
+// CHECK: }
+ gpu.func @min_number_vector(%arg0 : vector<2xf16>, %out : memref<vector<2xf16>>) kernel {
+ %result = gpu.subgroup_reduce minnumf %arg0 : (vector<2xf16>) -> vector<2xf16>
+ memref.store %result, %out[] : memref<vector<2xf16>>
+ gpu.return
+ }
+
+// CHECK-LABEL: gpu.func @max_number_vector(
+// CHECK-SAME: %[[ARG0:.*]]: vector<2xf16>,
+// CHECK-SAME: %[[ARG1:.*]]: memref<vector<2xf16>>) kernel {
+// CHECK: %[[CONSTANT_0:.*]] = arith.constant 16 : i32
+// CHECK: %[[CONSTANT_1:.*]] = arith.constant 8 : i32
+// CHECK: %[[CONSTANT_2:.*]] = arith.constant 4 : i32
+// CHECK: %[[CONSTANT_3:.*]] = arith.constant 2 : i32
+// CHECK: %[[CONSTANT_4:.*]] = arith.constant 32 : i32
+// CHECK: %[[CONSTANT_5:.*]] = arith.constant 1 : i32
+// CHECK: %[[BITCAST_0:.*]] = vector.bitcast %[[ARG0]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_0:.*]] = vector.extract %[[BITCAST_0]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[EXTRACT_0]], %[[CONSTANT_5]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_0:.*]] = vector.broadcast %[[VAL_0]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_1:.*]] = vector.bitcast %[[BROADCAST_0]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXNUMF_0:.*]] = arith.maxnumf %[[ARG0]], %[[BITCAST_1]] : vector<2xf16>
+// CHECK: %[[BITCAST_2:.*]] = vector.bitcast %[[MAXNUMF_0]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_1:.*]] = vector.extract %[[BITCAST_2]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_1:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[EXTRACT_1]], %[[CONSTANT_3]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_1:.*]] = vector.broadcast %[[VAL_1]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_3:.*]] = vector.bitcast %[[BROADCAST_1]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXNUMF_1:.*]] = arith.maxnumf %[[MAXNUMF_0]], %[[BITCAST_3]] : vector<2xf16>
+// CHECK: %[[BITCAST_4:.*]] = vector.bitcast %[[MAXNUMF_1]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_2:.*]] = vector.extract %[[BITCAST_4]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_2:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[EXTRACT_2]], %[[CONSTANT_2]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_2:.*]] = vector.broadcast %[[VAL_2]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_5:.*]] = vector.bitcast %[[BROADCAST_2]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXNUMF_2:.*]] = arith.maxnumf %[[MAXNUMF_1]], %[[BITCAST_5]] : vector<2xf16>
+// CHECK: %[[BITCAST_6:.*]] = vector.bitcast %[[MAXNUMF_2]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_3:.*]] = vector.extract %[[BITCAST_6]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_3:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[EXTRACT_3]], %[[CONSTANT_1]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_3:.*]] = vector.broadcast %[[VAL_3]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_7:.*]] = vector.bitcast %[[BROADCAST_3]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXNUMF_3:.*]] = arith.maxnumf %[[MAXNUMF_2]], %[[BITCAST_7]] : vector<2xf16>
+// CHECK: %[[BITCAST_8:.*]] = vector.bitcast %[[MAXNUMF_3]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_4:.*]] = vector.extract %[[BITCAST_8]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_4:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[EXTRACT_4]], %[[CONSTANT_0]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_4:.*]] = vector.broadcast %[[VAL_4]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_9:.*]] = vector.bitcast %[[BROADCAST_4]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXNUMF_4:.*]] = arith.maxnumf %[[MAXNUMF_3]], %[[BITCAST_9]] : vector<2xf16>
+// CHECK: memref.store %[[MAXNUMF_4]], %[[ARG1]][] : memref<vector<2xf16>>
+// CHECK: gpu.return
+// CHECK: }
+ gpu.func @max_number_vector(%arg0 : vector<2xf16>, %out : memref<vector<2xf16>>) kernel {
+ %result = gpu.subgroup_reduce maxnumf %arg0 : (vector<2xf16>) -> vector<2xf16>
+ memref.store %result, %out[] : memref<vector<2xf16>>
+ gpu.return
+ }
+}
diff --git a/mlir/test/Dialect/GPU/all-reduce-minimum-maximum.mlir b/mlir/test/Dialect/GPU/all-reduce-minimum-maximum.mlir
new file mode 100644
index 0000000000000..b3b826e3c0027
--- /dev/null
+++ b/mlir/test/Dialect/GPU/all-reduce-minimum-maximum.mlir
@@ -0,0 +1,466 @@
+// NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
+
+// RUN: mlir-opt -test-gpu-rewrite \
+// RUN: -test-gpu-subgroup-reduce-lowering=expand-to-shuffles %s | FileCheck %s
+
+gpu.module @kernels {
+// CHECK-LABEL: gpu.func @minimum_number(
+// CHECK-SAME: %[[ARG0:.*]]: f32,
+// CHECK-SAME: %[[ARG1:.*]]: memref<f32>) workgroup(
+// CHECK-SAME: %[[ARG2:.*]] : memref<32xf32, #gpu.address_space<workgroup>>) kernel {
+// CHECK: %[[CONSTANT_0:.*]] = arith.constant 0 : index
+// CHECK: %[[CONSTANT_1:.*]] = arith.constant 16 : i32
+// CHECK: %[[CONSTANT_2:.*]] = arith.constant 8 : i32
+// CHECK: %[[CONSTANT_3:.*]] = arith.constant 4 : i32
+// CHECK: %[[CONSTANT_4:.*]] = arith.constant 2 : i32
+// CHECK: %[[CONSTANT_5:.*]] = arith.constant 1 : i32
+// CHECK: %[[CONSTANT_6:.*]] = arith.constant 32 : i32
+// CHECK: %[[CONSTANT_7:.*]] = arith.constant 0 : i32
+// CHECK: %[[CONSTANT_8:.*]] = arith.constant 31 : i32
+// CHECK: %[[BLOCK_DIM_0:.*]] = gpu.block_dim x
+// CHECK: %[[INDEX_CAST_0:.*]] = arith.index_cast %[[BLOCK_DIM_0]] : index to i32
+// CHECK: %[[BLOCK_DIM_1:.*]] = gpu.block_dim y
+// CHECK: %[[INDEX_CAST_1:.*]] = arith.index_cast %[[BLOCK_DIM_1]] : index to i32
+// CHECK: %[[BLOCK_DIM_2:.*]] = gpu.block_dim z
+// CHECK: %[[INDEX_CAST_2:.*]] = arith.index_cast %[[BLOCK_DIM_2]] : index to i32
+// CHECK: %[[THREAD_ID_0:.*]] = gpu.thread_id x
+// CHECK: %[[INDEX_CAST_3:.*]] = arith.index_cast %[[THREAD_ID_0]] : index to i32
+// CHECK: %[[THREAD_ID_1:.*]] = gpu.thread_id y
+// CHECK: %[[INDEX_CAST_4:.*]] = arith.index_cast %[[THREAD_ID_1]] : index to i32
+// CHECK: %[[THREAD_ID_2:.*]] = gpu.thread_id z
+// CHECK: %[[INDEX_CAST_5:.*]] = arith.index_cast %[[THREAD_ID_2]] : index to i32
+// CHECK: %[[MULI_0:.*]] = arith.muli %[[INDEX_CAST_5]], %[[INDEX_CAST_1]] : i32
+// CHECK: %[[ADDI_0:.*]] = arith.addi %[[MULI_0]], %[[INDEX_CAST_4]] : i32
+// CHECK: %[[MULI_1:.*]] = arith.muli %[[ADDI_0]], %[[INDEX_CAST_0]] : i32
+// CHECK: %[[MULI_2:.*]] = arith.muli %[[INDEX_CAST_0]], %[[INDEX_CAST_1]] : i32
+// CHECK: %[[ADDI_1:.*]] = arith.addi %[[MULI_1]], %[[INDEX_CAST_3]] : i32
+// CHECK: %[[MULI_3:.*]] = arith.muli %[[MULI_2]], %[[INDEX_CAST_2]] : i32
+// CHECK: %[[ANDI_0:.*]] = arith.andi %[[ADDI_1]], %[[CONSTANT_8]] : i32
+// CHECK: %[[CMPI_0:.*]] = arith.cmpi eq, %[[ANDI_0]], %[[CONSTANT_7]] : i32
+// CHECK: %[[SUBI_0:.*]] = arith.subi %[[ADDI_1]], %[[ANDI_0]] : i32
+// CHECK: %[[SUBI_1:.*]] = arith.subi %[[MULI_3]], %[[SUBI_0]] : i32
+// CHECK: %[[CMPI_1:.*]] = arith.cmpi slt, %[[SUBI_1]], %[[CONSTANT_6]] : i32
+// CHECK: cf.cond_br %[[CMPI_1]], ^bb1, ^bb17
+// CHECK: ^bb1:
+// CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_0]], ^bb2, ^bb3
+// CHECK: ^bb2:
+// CHECK: %[[MINIMUMNUMF_0:.*]] = arith.minimumnumf %[[ARG0]], %[[VAL_0]] : f32
+// CHECK: cf.br ^bb4(%[[MINIMUMNUMF_0]] : f32)
+// CHECK: ^bb3:
+// CHECK: cf.br ^bb4(%[[ARG0]] : f32)
+// CHECK: ^bb4(%[[VAL_1:.*]]: f32):
+// CHECK: %[[VAL_2:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[VAL_1]], %[[CONSTANT_4]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_1]], ^bb5, ^bb6
+// CHECK: ^bb5:
+// CHECK: %[[MINIMUMNUMF_1:.*]] = arith.minimumnumf %[[VAL_1]], %[[VAL_2]] : f32
+// CHECK: cf.br ^bb7(%[[MINIMUMNUMF_1]] : f32)
+// CHECK: ^bb6:
+// CHECK: cf.br ^bb7(%[[VAL_1]] : f32)
+// CHECK: ^bb7(%[[VAL_3:.*]]: f32):
+// CHECK: %[[VAL_4:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[VAL_3]], %[[CONSTANT_3]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_2]], ^bb8, ^bb9
+// CHECK: ^bb8:
+// CHECK: %[[MINIMUMNUMF_2:.*]] = arith.minimumnumf %[[VAL_3]], %[[VAL_4]] : f32
+// CHECK: cf.br ^bb10(%[[MINIMUMNUMF_2]] : f32)
+// CHECK: ^bb9:
+// CHECK: cf.br ^bb10(%[[VAL_3]] : f32)
+// CHECK: ^bb10(%[[VAL_5:.*]]: f32):
+// CHECK: %[[VAL_6:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[VAL_5]], %[[CONSTANT_2]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_3]], ^bb11, ^bb12
+// CHECK: ^bb11:
+// CHECK: %[[MINIMUMNUMF_3:.*]] = arith.minimumnumf %[[VAL_5]], %[[VAL_6]] : f32
+// CHECK: cf.br ^bb13(%[[MINIMUMNUMF_3]] : f32)
+// CHECK: ^bb12:
+// CHECK: cf.br ^bb13(%[[VAL_5]] : f32)
+// CHECK: ^bb13(%[[VAL_7:.*]]: f32):
+// CHECK: %[[VAL_8:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[VAL_7]], %[[CONSTANT_1]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_4]], ^bb14, ^bb15
+// CHECK: ^bb14:
+// CHECK: %[[MINIMUMNUMF_4:.*]] = arith.minimumnumf %[[VAL_7]], %[[VAL_8]] : f32
+// CHECK: cf.br ^bb16(%[[MINIMUMNUMF_4]] : f32)
+// CHECK: ^bb15:
+// CHECK: cf.br ^bb16(%[[VAL_7]] : f32)
+// CHECK: ^bb16(%[[VAL_9:.*]]: f32):
+// CHECK: cf.br ^bb18(%[[VAL_9]] : f32)
+// CHECK: ^bb17:
+// CHECK: %[[VAL_10:.*]], %[[SHUFFLE_5:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_5:.*]] = arith.minimumnumf %[[ARG0]], %[[VAL_10]] : f32
+// CHECK: %[[VAL_11:.*]], %[[SHUFFLE_6:.*]] = gpu.shuffle xor %[[MINIMUMNUMF_5]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_6:.*]] = arith.minimumnumf %[[MINIMUMNUMF_5]], %[[VAL_11]] : f32
+// CHECK: %[[VAL_12:.*]], %[[SHUFFLE_7:.*]] = gpu.shuffle xor %[[MINIMUMNUMF_6]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_7:.*]] = arith.minimumnumf %[[MINIMUMNUMF_6]], %[[VAL_12]] : f32
+// CHECK: %[[VAL_13:.*]], %[[SHUFFLE_8:.*]] = gpu.shuffle xor %[[MINIMUMNUMF_7]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_8:.*]] = arith.minimumnumf %[[MINIMUMNUMF_7]], %[[VAL_13]] : f32
+// CHECK: %[[VAL_14:.*]], %[[SHUFFLE_9:.*]] = gpu.shuffle xor %[[MINIMUMNUMF_8]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_9:.*]] = arith.minimumnumf %[[MINIMUMNUMF_8]], %[[VAL_14]] : f32
+// CHECK: cf.br ^bb18(%[[MINIMUMNUMF_9]] : f32)
+// CHECK: ^bb18(%[[VAL_15:.*]]: f32):
+// CHECK: cf.cond_br %[[CMPI_0]], ^bb19, ^bb20
+// CHECK: ^bb19:
+// CHECK: %[[DIVSI_0:.*]] = arith.divsi %[[ADDI_1]], %[[CONSTANT_6]] : i32
+// CHECK: %[[INDEX_CAST_6:.*]] = arith.index_cast %[[DIVSI_0]] : i32 to index
+// CHECK: memref.store %[[VAL_15]], %[[ARG2]]{{\[}}%[[INDEX_CAST_6]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: cf.br ^bb21
+// CHECK: ^bb20:
+// CHECK: cf.br ^bb21
+// CHECK: ^bb21:
+// CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
+// CHECK: %[[ADDI_2:.*]] = arith.addi %[[MULI_3]], %[[CONSTANT_8]] : i32
+// CHECK: %[[DIVSI_1:.*]] = arith.divsi %[[ADDI_2]], %[[CONSTANT_6]] : i32
+// CHECK: %[[CMPI_2:.*]] = arith.cmpi slt, %[[ADDI_1]], %[[DIVSI_1]] : i32
+// CHECK: cf.cond_br %[[CMPI_2]], ^bb22, ^bb41
+// CHECK: ^bb22:
+// CHECK: %[[INDEX_CAST_7:.*]] = arith.index_cast %[[ADDI_1]] : i32 to index
+// CHECK: %[[LOAD_0:.*]] = memref.load %[[ARG2]]{{\[}}%[[INDEX_CAST_7]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: %[[CMPI_3:.*]] = arith.cmpi slt, %[[DIVSI_1]], %[[CONSTANT_6]] : i32
+// CHECK: cf.cond_br %[[CMPI_3]], ^bb23, ^bb39
+// CHECK: ^bb23:
+// CHECK: %[[VAL_16:.*]], %[[SHUFFLE_10:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_10]], ^bb24, ^bb25
+// CHECK: ^bb24:
+// CHECK: %[[MINIMUMNUMF_10:.*]] = arith.minimumnumf %[[LOAD_0]], %[[VAL_16]] : f32
+// CHECK: cf.br ^bb26(%[[MINIMUMNUMF_10]] : f32)
+// CHECK: ^bb25:
+// CHECK: cf.br ^bb26(%[[LOAD_0]] : f32)
+// CHECK: ^bb26(%[[VAL_17:.*]]: f32):
+// CHECK: %[[VAL_18:.*]], %[[SHUFFLE_11:.*]] = gpu.shuffle xor %[[VAL_17]], %[[CONSTANT_4]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_11]], ^bb27, ^bb28
+// CHECK: ^bb27:
+// CHECK: %[[MINIMUMNUMF_11:.*]] = arith.minimumnumf %[[VAL_17]], %[[VAL_18]] : f32
+// CHECK: cf.br ^bb29(%[[MINIMUMNUMF_11]] : f32)
+// CHECK: ^bb28:
+// CHECK: cf.br ^bb29(%[[VAL_17]] : f32)
+// CHECK: ^bb29(%[[VAL_19:.*]]: f32):
+// CHECK: %[[VAL_20:.*]], %[[SHUFFLE_12:.*]] = gpu.shuffle xor %[[VAL_19]], %[[CONSTANT_3]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_12]], ^bb30, ^bb31
+// CHECK: ^bb30:
+// CHECK: %[[MINIMUMNUMF_12:.*]] = arith.minimumnumf %[[VAL_19]], %[[VAL_20]] : f32
+// CHECK: cf.br ^bb32(%[[MINIMUMNUMF_12]] : f32)
+// CHECK: ^bb31:
+// CHECK: cf.br ^bb32(%[[VAL_19]] : f32)
+// CHECK: ^bb32(%[[VAL_21:.*]]: f32):
+// CHECK: %[[VAL_22:.*]], %[[SHUFFLE_13:.*]] = gpu.shuffle xor %[[VAL_21]], %[[CONSTANT_2]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_13]], ^bb33, ^bb34
+// CHECK: ^bb33:
+// CHECK: %[[MINIMUMNUMF_13:.*]] = arith.minimumnumf %[[VAL_21]], %[[VAL_22]] : f32
+// CHECK: cf.br ^bb35(%[[MINIMUMNUMF_13]] : f32)
+// CHECK: ^bb34:
+// CHECK: cf.br ^bb35(%[[VAL_21]] : f32)
+// CHECK: ^bb35(%[[VAL_23:.*]]: f32):
+// CHECK: %[[VAL_24:.*]], %[[SHUFFLE_14:.*]] = gpu.shuffle xor %[[VAL_23]], %[[CONSTANT_1]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_14]], ^bb36, ^bb37
+// CHECK: ^bb36:
+// CHECK: %[[MINIMUMNUMF_14:.*]] = arith.minimumnumf %[[VAL_23]], %[[VAL_24]] : f32
+// CHECK: cf.br ^bb38(%[[MINIMUMNUMF_14]] : f32)
+// CHECK: ^bb37:
+// CHECK: cf.br ^bb38(%[[VAL_23]] : f32)
+// CHECK: ^bb38(%[[VAL_25:.*]]: f32):
+// CHECK: cf.br ^bb40(%[[VAL_25]] : f32)
+// CHECK: ^bb39:
+// CHECK: %[[VAL_26:.*]], %[[SHUFFLE_15:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_15:.*]] = arith.minimumnumf %[[LOAD_0]], %[[VAL_26]] : f32
+// CHECK: %[[VAL_27:.*]], %[[SHUFFLE_16:.*]] = gpu.shuffle xor %[[MINIMUMNUMF_15]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_16:.*]] = arith.minimumnumf %[[MINIMUMNUMF_15]], %[[VAL_27]] : f32
+// CHECK: %[[VAL_28:.*]], %[[SHUFFLE_17:.*]] = gpu.shuffle xor %[[MINIMUMNUMF_16]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_17:.*]] = arith.minimumnumf %[[MINIMUMNUMF_16]], %[[VAL_28]] : f32
+// CHECK: %[[VAL_29:.*]], %[[SHUFFLE_18:.*]] = gpu.shuffle xor %[[MINIMUMNUMF_17]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_18:.*]] = arith.minimumnumf %[[MINIMUMNUMF_17]], %[[VAL_29]] : f32
+// CHECK: %[[VAL_30:.*]], %[[SHUFFLE_19:.*]] = gpu.shuffle xor %[[MINIMUMNUMF_18]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MINIMUMNUMF_19:.*]] = arith.minimumnumf %[[MINIMUMNUMF_18]], %[[VAL_30]] : f32
+// CHECK: cf.br ^bb40(%[[MINIMUMNUMF_19]] : f32)
+// CHECK: ^bb40(%[[VAL_31:.*]]: f32):
+// CHECK: memref.store %[[VAL_31]], %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: cf.br ^bb42
+// CHECK: ^bb41:
+// CHECK: cf.br ^bb42
+// CHECK: ^bb42:
+// CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
+// CHECK: %[[LOAD_1:.*]] = memref.load %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: memref.store %[[LOAD_1]], %[[ARG1]][] : memref<f32>
+// CHECK: gpu.return
+// CHECK: }
+ gpu.func @minimum_number(%arg0 : f32, %out : memref<f32>) kernel {
+ %result = gpu.all_reduce minimumnumf %arg0 uniform {} : (f32) -> f32
+ memref.store %result, %out[] : memref<f32>
+ gpu.return
+ }
+
+// CHECK-LABEL: gpu.func @maximum_number(
+// CHECK-SAME: %[[ARG0:.*]]: f32,
+// CHECK-SAME: %[[ARG1:.*]]: memref<f32>) workgroup(
+// CHECK-SAME: %[[ARG2:.*]] : memref<32xf32, #gpu.address_space<workgroup>>) kernel {
+// CHECK: %[[CONSTANT_0:.*]] = arith.constant 0 : index
+// CHECK: %[[CONSTANT_1:.*]] = arith.constant 16 : i32
+// CHECK: %[[CONSTANT_2:.*]] = arith.constant 8 : i32
+// CHECK: %[[CONSTANT_3:.*]] = arith.constant 4 : i32
+// CHECK: %[[CONSTANT_4:.*]] = arith.constant 2 : i32
+// CHECK: %[[CONSTANT_5:.*]] = arith.constant 1 : i32
+// CHECK: %[[CONSTANT_6:.*]] = arith.constant 32 : i32
+// CHECK: %[[CONSTANT_7:.*]] = arith.constant 0 : i32
+// CHECK: %[[CONSTANT_8:.*]] = arith.constant 31 : i32
+// CHECK: %[[BLOCK_DIM_0:.*]] = gpu.block_dim x
+// CHECK: %[[INDEX_CAST_0:.*]] = arith.index_cast %[[BLOCK_DIM_0]] : index to i32
+// CHECK: %[[BLOCK_DIM_1:.*]] = gpu.block_dim y
+// CHECK: %[[INDEX_CAST_1:.*]] = arith.index_cast %[[BLOCK_DIM_1]] : index to i32
+// CHECK: %[[BLOCK_DIM_2:.*]] = gpu.block_dim z
+// CHECK: %[[INDEX_CAST_2:.*]] = arith.index_cast %[[BLOCK_DIM_2]] : index to i32
+// CHECK: %[[THREAD_ID_0:.*]] = gpu.thread_id x
+// CHECK: %[[INDEX_CAST_3:.*]] = arith.index_cast %[[THREAD_ID_0]] : index to i32
+// CHECK: %[[THREAD_ID_1:.*]] = gpu.thread_id y
+// CHECK: %[[INDEX_CAST_4:.*]] = arith.index_cast %[[THREAD_ID_1]] : index to i32
+// CHECK: %[[THREAD_ID_2:.*]] = gpu.thread_id z
+// CHECK: %[[INDEX_CAST_5:.*]] = arith.index_cast %[[THREAD_ID_2]] : index to i32
+// CHECK: %[[MULI_0:.*]] = arith.muli %[[INDEX_CAST_5]], %[[INDEX_CAST_1]] : i32
+// CHECK: %[[ADDI_0:.*]] = arith.addi %[[MULI_0]], %[[INDEX_CAST_4]] : i32
+// CHECK: %[[MULI_1:.*]] = arith.muli %[[ADDI_0]], %[[INDEX_CAST_0]] : i32
+// CHECK: %[[MULI_2:.*]] = arith.muli %[[INDEX_CAST_0]], %[[INDEX_CAST_1]] : i32
+// CHECK: %[[ADDI_1:.*]] = arith.addi %[[MULI_1]], %[[INDEX_CAST_3]] : i32
+// CHECK: %[[MULI_3:.*]] = arith.muli %[[MULI_2]], %[[INDEX_CAST_2]] : i32
+// CHECK: %[[ANDI_0:.*]] = arith.andi %[[ADDI_1]], %[[CONSTANT_8]] : i32
+// CHECK: %[[CMPI_0:.*]] = arith.cmpi eq, %[[ANDI_0]], %[[CONSTANT_7]] : i32
+// CHECK: %[[SUBI_0:.*]] = arith.subi %[[ADDI_1]], %[[ANDI_0]] : i32
+// CHECK: %[[SUBI_1:.*]] = arith.subi %[[MULI_3]], %[[SUBI_0]] : i32
+// CHECK: %[[CMPI_1:.*]] = arith.cmpi slt, %[[SUBI_1]], %[[CONSTANT_6]] : i32
+// CHECK: cf.cond_br %[[CMPI_1]], ^bb1, ^bb17
+// CHECK: ^bb1:
+// CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_0]], ^bb2, ^bb3
+// CHECK: ^bb2:
+// CHECK: %[[MAXIMUMNUMF_0:.*]] = arith.maximumnumf %[[ARG0]], %[[VAL_0]] : f32
+// CHECK: cf.br ^bb4(%[[MAXIMUMNUMF_0]] : f32)
+// CHECK: ^bb3:
+// CHECK: cf.br ^bb4(%[[ARG0]] : f32)
+// CHECK: ^bb4(%[[VAL_1:.*]]: f32):
+// CHECK: %[[VAL_2:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[VAL_1]], %[[CONSTANT_4]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_1]], ^bb5, ^bb6
+// CHECK: ^bb5:
+// CHECK: %[[MAXIMUMNUMF_1:.*]] = arith.maximumnumf %[[VAL_1]], %[[VAL_2]] : f32
+// CHECK: cf.br ^bb7(%[[MAXIMUMNUMF_1]] : f32)
+// CHECK: ^bb6:
+// CHECK: cf.br ^bb7(%[[VAL_1]] : f32)
+// CHECK: ^bb7(%[[VAL_3:.*]]: f32):
+// CHECK: %[[VAL_4:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[VAL_3]], %[[CONSTANT_3]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_2]], ^bb8, ^bb9
+// CHECK: ^bb8:
+// CHECK: %[[MAXIMUMNUMF_2:.*]] = arith.maximumnumf %[[VAL_3]], %[[VAL_4]] : f32
+// CHECK: cf.br ^bb10(%[[MAXIMUMNUMF_2]] : f32)
+// CHECK: ^bb9:
+// CHECK: cf.br ^bb10(%[[VAL_3]] : f32)
+// CHECK: ^bb10(%[[VAL_5:.*]]: f32):
+// CHECK: %[[VAL_6:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[VAL_5]], %[[CONSTANT_2]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_3]], ^bb11, ^bb12
+// CHECK: ^bb11:
+// CHECK: %[[MAXIMUMNUMF_3:.*]] = arith.maximumnumf %[[VAL_5]], %[[VAL_6]] : f32
+// CHECK: cf.br ^bb13(%[[MAXIMUMNUMF_3]] : f32)
+// CHECK: ^bb12:
+// CHECK: cf.br ^bb13(%[[VAL_5]] : f32)
+// CHECK: ^bb13(%[[VAL_7:.*]]: f32):
+// CHECK: %[[VAL_8:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[VAL_7]], %[[CONSTANT_1]], %[[SUBI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_4]], ^bb14, ^bb15
+// CHECK: ^bb14:
+// CHECK: %[[MAXIMUMNUMF_4:.*]] = arith.maximumnumf %[[VAL_7]], %[[VAL_8]] : f32
+// CHECK: cf.br ^bb16(%[[MAXIMUMNUMF_4]] : f32)
+// CHECK: ^bb15:
+// CHECK: cf.br ^bb16(%[[VAL_7]] : f32)
+// CHECK: ^bb16(%[[VAL_9:.*]]: f32):
+// CHECK: cf.br ^bb18(%[[VAL_9]] : f32)
+// CHECK: ^bb17:
+// CHECK: %[[VAL_10:.*]], %[[SHUFFLE_5:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_5:.*]] = arith.maximumnumf %[[ARG0]], %[[VAL_10]] : f32
+// CHECK: %[[VAL_11:.*]], %[[SHUFFLE_6:.*]] = gpu.shuffle xor %[[MAXIMUMNUMF_5]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_6:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_5]], %[[VAL_11]] : f32
+// CHECK: %[[VAL_12:.*]], %[[SHUFFLE_7:.*]] = gpu.shuffle xor %[[MAXIMUMNUMF_6]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_7:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_6]], %[[VAL_12]] : f32
+// CHECK: %[[VAL_13:.*]], %[[SHUFFLE_8:.*]] = gpu.shuffle xor %[[MAXIMUMNUMF_7]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_8:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_7]], %[[VAL_13]] : f32
+// CHECK: %[[VAL_14:.*]], %[[SHUFFLE_9:.*]] = gpu.shuffle xor %[[MAXIMUMNUMF_8]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_9:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_8]], %[[VAL_14]] : f32
+// CHECK: cf.br ^bb18(%[[MAXIMUMNUMF_9]] : f32)
+// CHECK: ^bb18(%[[VAL_15:.*]]: f32):
+// CHECK: cf.cond_br %[[CMPI_0]], ^bb19, ^bb20
+// CHECK: ^bb19:
+// CHECK: %[[DIVSI_0:.*]] = arith.divsi %[[ADDI_1]], %[[CONSTANT_6]] : i32
+// CHECK: %[[INDEX_CAST_6:.*]] = arith.index_cast %[[DIVSI_0]] : i32 to index
+// CHECK: memref.store %[[VAL_15]], %[[ARG2]]{{\[}}%[[INDEX_CAST_6]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: cf.br ^bb21
+// CHECK: ^bb20:
+// CHECK: cf.br ^bb21
+// CHECK: ^bb21:
+// CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
+// CHECK: %[[ADDI_2:.*]] = arith.addi %[[MULI_3]], %[[CONSTANT_8]] : i32
+// CHECK: %[[DIVSI_1:.*]] = arith.divsi %[[ADDI_2]], %[[CONSTANT_6]] : i32
+// CHECK: %[[CMPI_2:.*]] = arith.cmpi slt, %[[ADDI_1]], %[[DIVSI_1]] : i32
+// CHECK: cf.cond_br %[[CMPI_2]], ^bb22, ^bb41
+// CHECK: ^bb22:
+// CHECK: %[[INDEX_CAST_7:.*]] = arith.index_cast %[[ADDI_1]] : i32 to index
+// CHECK: %[[LOAD_0:.*]] = memref.load %[[ARG2]]{{\[}}%[[INDEX_CAST_7]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: %[[CMPI_3:.*]] = arith.cmpi slt, %[[DIVSI_1]], %[[CONSTANT_6]] : i32
+// CHECK: cf.cond_br %[[CMPI_3]], ^bb23, ^bb39
+// CHECK: ^bb23:
+// CHECK: %[[VAL_16:.*]], %[[SHUFFLE_10:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_10]], ^bb24, ^bb25
+// CHECK: ^bb24:
+// CHECK: %[[MAXIMUMNUMF_10:.*]] = arith.maximumnumf %[[LOAD_0]], %[[VAL_16]] : f32
+// CHECK: cf.br ^bb26(%[[MAXIMUMNUMF_10]] : f32)
+// CHECK: ^bb25:
+// CHECK: cf.br ^bb26(%[[LOAD_0]] : f32)
+// CHECK: ^bb26(%[[VAL_17:.*]]: f32):
+// CHECK: %[[VAL_18:.*]], %[[SHUFFLE_11:.*]] = gpu.shuffle xor %[[VAL_17]], %[[CONSTANT_4]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_11]], ^bb27, ^bb28
+// CHECK: ^bb27:
+// CHECK: %[[MAXIMUMNUMF_11:.*]] = arith.maximumnumf %[[VAL_17]], %[[VAL_18]] : f32
+// CHECK: cf.br ^bb29(%[[MAXIMUMNUMF_11]] : f32)
+// CHECK: ^bb28:
+// CHECK: cf.br ^bb29(%[[VAL_17]] : f32)
+// CHECK: ^bb29(%[[VAL_19:.*]]: f32):
+// CHECK: %[[VAL_20:.*]], %[[SHUFFLE_12:.*]] = gpu.shuffle xor %[[VAL_19]], %[[CONSTANT_3]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_12]], ^bb30, ^bb31
+// CHECK: ^bb30:
+// CHECK: %[[MAXIMUMNUMF_12:.*]] = arith.maximumnumf %[[VAL_19]], %[[VAL_20]] : f32
+// CHECK: cf.br ^bb32(%[[MAXIMUMNUMF_12]] : f32)
+// CHECK: ^bb31:
+// CHECK: cf.br ^bb32(%[[VAL_19]] : f32)
+// CHECK: ^bb32(%[[VAL_21:.*]]: f32):
+// CHECK: %[[VAL_22:.*]], %[[SHUFFLE_13:.*]] = gpu.shuffle xor %[[VAL_21]], %[[CONSTANT_2]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_13]], ^bb33, ^bb34
+// CHECK: ^bb33:
+// CHECK: %[[MAXIMUMNUMF_13:.*]] = arith.maximumnumf %[[VAL_21]], %[[VAL_22]] : f32
+// CHECK: cf.br ^bb35(%[[MAXIMUMNUMF_13]] : f32)
+// CHECK: ^bb34:
+// CHECK: cf.br ^bb35(%[[VAL_21]] : f32)
+// CHECK: ^bb35(%[[VAL_23:.*]]: f32):
+// CHECK: %[[VAL_24:.*]], %[[SHUFFLE_14:.*]] = gpu.shuffle xor %[[VAL_23]], %[[CONSTANT_1]], %[[DIVSI_1]] : f32
+// CHECK: cf.cond_br %[[SHUFFLE_14]], ^bb36, ^bb37
+// CHECK: ^bb36:
+// CHECK: %[[MAXIMUMNUMF_14:.*]] = arith.maximumnumf %[[VAL_23]], %[[VAL_24]] : f32
+// CHECK: cf.br ^bb38(%[[MAXIMUMNUMF_14]] : f32)
+// CHECK: ^bb37:
+// CHECK: cf.br ^bb38(%[[VAL_23]] : f32)
+// CHECK: ^bb38(%[[VAL_25:.*]]: f32):
+// CHECK: cf.br ^bb40(%[[VAL_25]] : f32)
+// CHECK: ^bb39:
+// CHECK: %[[VAL_26:.*]], %[[SHUFFLE_15:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_15:.*]] = arith.maximumnumf %[[LOAD_0]], %[[VAL_26]] : f32
+// CHECK: %[[VAL_27:.*]], %[[SHUFFLE_16:.*]] = gpu.shuffle xor %[[MAXIMUMNUMF_15]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_16:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_15]], %[[VAL_27]] : f32
+// CHECK: %[[VAL_28:.*]], %[[SHUFFLE_17:.*]] = gpu.shuffle xor %[[MAXIMUMNUMF_16]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_17:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_16]], %[[VAL_28]] : f32
+// CHECK: %[[VAL_29:.*]], %[[SHUFFLE_18:.*]] = gpu.shuffle xor %[[MAXIMUMNUMF_17]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_18:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_17]], %[[VAL_29]] : f32
+// CHECK: %[[VAL_30:.*]], %[[SHUFFLE_19:.*]] = gpu.shuffle xor %[[MAXIMUMNUMF_18]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32
+// CHECK: %[[MAXIMUMNUMF_19:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_18]], %[[VAL_30]] : f32
+// CHECK: cf.br ^bb40(%[[MAXIMUMNUMF_19]] : f32)
+// CHECK: ^bb40(%[[VAL_31:.*]]: f32):
+// CHECK: memref.store %[[VAL_31]], %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: cf.br ^bb42
+// CHECK: ^bb41:
+// CHECK: cf.br ^bb42
+// CHECK: ^bb42:
+// CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>]
+// CHECK: %[[LOAD_1:.*]] = memref.load %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>>
+// CHECK: memref.store %[[LOAD_1]], %[[ARG1]][] : memref<f32>
+// CHECK: gpu.return
+// CHECK: }
+ gpu.func @maximum_number(%arg0 : f32, %out : memref<f32>) kernel {
+ %result = gpu.all_reduce maximumnumf %arg0 uniform {} : (f32) -> f32
+ memref.store %result, %out[] : memref<f32>
+ gpu.return
+ }
+
+// CHECK-LABEL: gpu.func @minimum_number_vector(
+// CHECK-SAME: %[[ARG0:.*]]: vector<2xf16>,
+// CHECK-SAME: %[[ARG1:.*]]: memref<vector<2xf16>>) kernel {
+// CHECK: %[[CONSTANT_0:.*]] = arith.constant 16 : i32
+// CHECK: %[[CONSTANT_1:.*]] = arith.constant 8 : i32
+// CHECK: %[[CONSTANT_2:.*]] = arith.constant 4 : i32
+// CHECK: %[[CONSTANT_3:.*]] = arith.constant 2 : i32
+// CHECK: %[[CONSTANT_4:.*]] = arith.constant 32 : i32
+// CHECK: %[[CONSTANT_5:.*]] = arith.constant 1 : i32
+// CHECK: %[[BITCAST_0:.*]] = vector.bitcast %[[ARG0]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_0:.*]] = vector.extract %[[BITCAST_0]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[EXTRACT_0]], %[[CONSTANT_5]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_0:.*]] = vector.broadcast %[[VAL_0]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_1:.*]] = vector.bitcast %[[BROADCAST_0]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINIMUMNUMF_0:.*]] = arith.minimumnumf %[[ARG0]], %[[BITCAST_1]] : vector<2xf16>
+// CHECK: %[[BITCAST_2:.*]] = vector.bitcast %[[MINIMUMNUMF_0]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_1:.*]] = vector.extract %[[BITCAST_2]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_1:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[EXTRACT_1]], %[[CONSTANT_3]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_1:.*]] = vector.broadcast %[[VAL_1]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_3:.*]] = vector.bitcast %[[BROADCAST_1]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINIMUMNUMF_1:.*]] = arith.minimumnumf %[[MINIMUMNUMF_0]], %[[BITCAST_3]] : vector<2xf16>
+// CHECK: %[[BITCAST_4:.*]] = vector.bitcast %[[MINIMUMNUMF_1]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_2:.*]] = vector.extract %[[BITCAST_4]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_2:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[EXTRACT_2]], %[[CONSTANT_2]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_2:.*]] = vector.broadcast %[[VAL_2]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_5:.*]] = vector.bitcast %[[BROADCAST_2]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINIMUMNUMF_2:.*]] = arith.minimumnumf %[[MINIMUMNUMF_1]], %[[BITCAST_5]] : vector<2xf16>
+// CHECK: %[[BITCAST_6:.*]] = vector.bitcast %[[MINIMUMNUMF_2]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_3:.*]] = vector.extract %[[BITCAST_6]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_3:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[EXTRACT_3]], %[[CONSTANT_1]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_3:.*]] = vector.broadcast %[[VAL_3]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_7:.*]] = vector.bitcast %[[BROADCAST_3]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINIMUMNUMF_3:.*]] = arith.minimumnumf %[[MINIMUMNUMF_2]], %[[BITCAST_7]] : vector<2xf16>
+// CHECK: %[[BITCAST_8:.*]] = vector.bitcast %[[MINIMUMNUMF_3]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_4:.*]] = vector.extract %[[BITCAST_8]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_4:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[EXTRACT_4]], %[[CONSTANT_0]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_4:.*]] = vector.broadcast %[[VAL_4]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_9:.*]] = vector.bitcast %[[BROADCAST_4]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MINIMUMNUMF_4:.*]] = arith.minimumnumf %[[MINIMUMNUMF_3]], %[[BITCAST_9]] : vector<2xf16>
+// CHECK: memref.store %[[MINIMUMNUMF_4]], %[[ARG1]][] : memref<vector<2xf16>>
+// CHECK: gpu.return
+// CHECK: }
+ gpu.func @minimum_number_vector(%arg0 : vector<2xf16>, %out : memref<vector<2xf16>>) kernel {
+ %result = gpu.subgroup_reduce minimumnumf %arg0 : (vector<2xf16>) -> vector<2xf16>
+ memref.store %result, %out[] : memref<vector<2xf16>>
+ gpu.return
+ }
+
+// CHECK-LABEL: gpu.func @maximum_number_vector(
+// CHECK-SAME: %[[ARG0:.*]]: vector<2xf16>,
+// CHECK-SAME: %[[ARG1:.*]]: memref<vector<2xf16>>) kernel {
+// CHECK: %[[CONSTANT_0:.*]] = arith.constant 16 : i32
+// CHECK: %[[CONSTANT_1:.*]] = arith.constant 8 : i32
+// CHECK: %[[CONSTANT_2:.*]] = arith.constant 4 : i32
+// CHECK: %[[CONSTANT_3:.*]] = arith.constant 2 : i32
+// CHECK: %[[CONSTANT_4:.*]] = arith.constant 32 : i32
+// CHECK: %[[CONSTANT_5:.*]] = arith.constant 1 : i32
+// CHECK: %[[BITCAST_0:.*]] = vector.bitcast %[[ARG0]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_0:.*]] = vector.extract %[[BITCAST_0]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[EXTRACT_0]], %[[CONSTANT_5]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_0:.*]] = vector.broadcast %[[VAL_0]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_1:.*]] = vector.bitcast %[[BROADCAST_0]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXIMUMNUMF_0:.*]] = arith.maximumnumf %[[ARG0]], %[[BITCAST_1]] : vector<2xf16>
+// CHECK: %[[BITCAST_2:.*]] = vector.bitcast %[[MAXIMUMNUMF_0]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_1:.*]] = vector.extract %[[BITCAST_2]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_1:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[EXTRACT_1]], %[[CONSTANT_3]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_1:.*]] = vector.broadcast %[[VAL_1]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_3:.*]] = vector.bitcast %[[BROADCAST_1]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXIMUMNUMF_1:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_0]], %[[BITCAST_3]] : vector<2xf16>
+// CHECK: %[[BITCAST_4:.*]] = vector.bitcast %[[MAXIMUMNUMF_1]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_2:.*]] = vector.extract %[[BITCAST_4]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_2:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[EXTRACT_2]], %[[CONSTANT_2]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_2:.*]] = vector.broadcast %[[VAL_2]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_5:.*]] = vector.bitcast %[[BROADCAST_2]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXIMUMNUMF_2:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_1]], %[[BITCAST_5]] : vector<2xf16>
+// CHECK: %[[BITCAST_6:.*]] = vector.bitcast %[[MAXIMUMNUMF_2]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_3:.*]] = vector.extract %[[BITCAST_6]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_3:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[EXTRACT_3]], %[[CONSTANT_1]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_3:.*]] = vector.broadcast %[[VAL_3]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_7:.*]] = vector.bitcast %[[BROADCAST_3]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXIMUMNUMF_3:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_2]], %[[BITCAST_7]] : vector<2xf16>
+// CHECK: %[[BITCAST_8:.*]] = vector.bitcast %[[MAXIMUMNUMF_3]] : vector<2xf16> to vector<1xi32>
+// CHECK: %[[EXTRACT_4:.*]] = vector.extract %[[BITCAST_8]][0] : i32 from vector<1xi32>
+// CHECK: %[[VAL_4:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[EXTRACT_4]], %[[CONSTANT_0]], %[[CONSTANT_4]] : i32
+// CHECK: %[[BROADCAST_4:.*]] = vector.broadcast %[[VAL_4]] : i32 to vector<1xi32>
+// CHECK: %[[BITCAST_9:.*]] = vector.bitcast %[[BROADCAST_4]] : vector<1xi32> to vector<2xf16>
+// CHECK: %[[MAXIMUMNUMF_4:.*]] = arith.maximumnumf %[[MAXIMUMNUMF_3]], %[[BITCAST_9]] : vector<2xf16>
+// CHECK: memref.store %[[MAXIMUMNUMF_4]], %[[ARG1]][] : memref<vector<2xf16>>
+// CHECK: gpu.return
+// CHECK: }
+ gpu.func @maximum_number_vector(%arg0 : vector<2xf16>, %out : memref<vector<2xf16>>) kernel {
+ %result = gpu.subgroup_reduce maximumnumf %arg0 : (vector<2xf16>) -> vector<2xf16>
+ memref.store %result, %out[] : memref<vector<2xf16>>
+ gpu.return
+ }
+}
diff --git a/mlir/test/Dialect/GPU/all-reduce-minmaxnumf.mlir b/mlir/test/Dialect/GPU/all-reduce-minmaxnumf.mlir
deleted file mode 100644
index 7ee88d38ffd4f..0000000000000
--- a/mlir/test/Dialect/GPU/all-reduce-minmaxnumf.mlir
+++ /dev/null
@@ -1,21 +0,0 @@
-// RUN: mlir-opt --convert-gpu-to-nvvm %s | FileCheck %s
-
-gpu.module @kernels {
- // CHECK-LABEL: llvm.func @minimum_number
- // CHECK: llvm.intr.minimumnum
- // CHECK: llvm.return
- gpu.func @minimum_number(%arg0 : f32, %out : memref<f32>) kernel {
- %result = gpu.all_reduce minimumnumf %arg0 uniform {} : (f32) -> f32
- memref.store %result, %out[] : memref<f32>
- gpu.return
- }
-
- // CHECK-LABEL: llvm.func @maximum_number
- // CHECK: llvm.intr.maximumnum
- // CHECK: llvm.return
- gpu.func @maximum_number(%arg0 : f32, %out : memref<f32>) kernel {
- %result = gpu.all_reduce maximumnumf %arg0 uniform {} : (f32) -> f32
- memref.store %result, %out[] : memref<f32>
- gpu.return
- }
-}
More information about the Mlir-commits
mailing list