[llvm-branch-commits] [clang] [libclc] [llvm] [mlir] [CIR] Remove cir.unary(plus, ...) and emit nothing for unary plus (PR #185199)
Henrich Lauko via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Mar 8 06:45:16 PDT 2026
https://github.com/xlauko updated https://github.com/llvm/llvm-project/pull/185199
>From 80ff7369c9ecdbef23a95b234eef3f05e7c37575 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 08:42:25 +0100
Subject: [PATCH 01/25] libclc: Add __printf_alloc implementation (#185231)
AMDGPU OpenCL printf implementation emits a call to this helper
function.
---
libclc/opencl/lib/amdgcn/SOURCES | 1 +
.../lib/amdgcn/printf/__printf_alloc.cl | 36 +++++++++++++++++++
2 files changed, 37 insertions(+)
create mode 100644 libclc/opencl/lib/amdgcn/printf/__printf_alloc.cl
diff --git a/libclc/opencl/lib/amdgcn/SOURCES b/libclc/opencl/lib/amdgcn/SOURCES
index 4153d964e118d..caf08495e8dae 100644
--- a/libclc/opencl/lib/amdgcn/SOURCES
+++ b/libclc/opencl/lib/amdgcn/SOURCES
@@ -1,4 +1,5 @@
async/wait_group_events.cl
mem_fence/fence.cl
+printf/__printf_alloc.cl
subgroup/subgroup.cl
synchronization/sub_group_barrier.cl
diff --git a/libclc/opencl/lib/amdgcn/printf/__printf_alloc.cl b/libclc/opencl/lib/amdgcn/printf/__printf_alloc.cl
new file mode 100644
index 0000000000000..f4a7f46cbbcac
--- /dev/null
+++ b/libclc/opencl/lib/amdgcn/printf/__printf_alloc.cl
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <amdhsa_abi.h>
+
+#define OFFSET 8
+
+// Atomically reserves space to the printf data buffer and returns a pointer to
+// it
+__global char *__printf_alloc(uint bytes) {
+ __constant amdhsa_implicit_kernarg_v5 *args =
+ (__constant amdhsa_implicit_kernarg_v5 *)
+ __builtin_amdgcn_implicitarg_ptr();
+ __global char *ptr = (__global char *)args->printf_buffer;
+
+ uint size = ((__global uint *)ptr)[1];
+ uint offset = __opencl_atomic_load((__global atomic_uint *)ptr,
+ memory_order_relaxed, memory_scope_device);
+
+ for (;;) {
+ if (OFFSET + offset + bytes > size)
+ return NULL;
+
+ if (__opencl_atomic_compare_exchange_strong(
+ (__global atomic_uint *)ptr, &offset, offset + bytes,
+ memory_order_relaxed, memory_order_relaxed, memory_scope_device))
+ break;
+ }
+
+ return ptr + OFFSET + offset;
+}
>From b7403433f8dbd53f7e97afbd6587432dbe00cc18 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 08:43:17 +0100
Subject: [PATCH 02/25] libclc: Move get_global_id into clc (#185180)
---
libclc/clc/lib/generic/SOURCES | 1 +
.../lib/generic/workitem/clc_get_global_id.cl | 17 +++++++++++++++++
.../lib/generic/workitem/get_global_id.cl | 5 ++---
3 files changed, 20 insertions(+), 3 deletions(-)
create mode 100644 libclc/clc/lib/generic/workitem/clc_get_global_id.cl
diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES
index b352382fbe6d9..926efe9c6f188 100644
--- a/libclc/clc/lib/generic/SOURCES
+++ b/libclc/clc/lib/generic/SOURCES
@@ -176,6 +176,7 @@ shared/clc_min.cl
shared/clc_qualifier.cl
shared/clc_vload.cl
shared/clc_vstore.cl
+workitem/clc_get_global_id.cl
workitem/clc_get_global_linear_id.cl
workitem/clc_get_local_linear_id.cl
workitem/clc_get_num_sub_groups.cl
diff --git a/libclc/clc/lib/generic/workitem/clc_get_global_id.cl b/libclc/clc/lib/generic/workitem/clc_get_global_id.cl
new file mode 100644
index 0000000000000..515ef7a173f25
--- /dev/null
+++ b/libclc/clc/lib/generic/workitem/clc_get_global_id.cl
@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/workitem/clc_get_global_offset.h>
+#include <clc/workitem/clc_get_group_id.h>
+#include <clc/workitem/clc_get_local_id.h>
+#include <clc/workitem/clc_get_local_size.h>
+
+_CLC_OVERLOAD _CLC_DEF size_t __clc_get_global_id(uint dim) {
+ return __clc_get_group_id(dim) * __clc_get_local_size(dim) +
+ __clc_get_local_id(dim) + __clc_get_global_offset(dim);
+}
diff --git a/libclc/opencl/lib/generic/workitem/get_global_id.cl b/libclc/opencl/lib/generic/workitem/get_global_id.cl
index f56981975acd3..d134dfc3f8d72 100644
--- a/libclc/opencl/lib/generic/workitem/get_global_id.cl
+++ b/libclc/opencl/lib/generic/workitem/get_global_id.cl
@@ -6,9 +6,8 @@
//
//===----------------------------------------------------------------------===//
-#include <clc/opencl/opencl-base.h>
+#include <clc/workitem/clc_get_global_id.h>
_CLC_DEF _CLC_OVERLOAD size_t get_global_id(uint dim) {
- return get_group_id(dim) * get_local_size(dim) + get_local_id(dim) +
- get_global_offset(dim);
+ return __clc_get_global_id(dim);
}
>From 592c0f5d74fd6090ae6c278540954d612669e826 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 08:48:25 +0100
Subject: [PATCH 03/25] libclc: Use enqueued local size to implement
get_global_id (#185181)
---
libclc/clc/lib/generic/workitem/clc_get_global_id.cl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/libclc/clc/lib/generic/workitem/clc_get_global_id.cl b/libclc/clc/lib/generic/workitem/clc_get_global_id.cl
index 515ef7a173f25..e3da8fd48fa51 100644
--- a/libclc/clc/lib/generic/workitem/clc_get_global_id.cl
+++ b/libclc/clc/lib/generic/workitem/clc_get_global_id.cl
@@ -6,12 +6,12 @@
//
//===----------------------------------------------------------------------===//
+#include <clc/workitem/clc_get_enqueued_local_size.h>
#include <clc/workitem/clc_get_global_offset.h>
#include <clc/workitem/clc_get_group_id.h>
#include <clc/workitem/clc_get_local_id.h>
-#include <clc/workitem/clc_get_local_size.h>
_CLC_OVERLOAD _CLC_DEF size_t __clc_get_global_id(uint dim) {
- return __clc_get_group_id(dim) * __clc_get_local_size(dim) +
+ return __clc_get_group_id(dim) * __clc_get_enqueued_local_size(dim) +
__clc_get_local_id(dim) + __clc_get_global_offset(dim);
}
>From 65775204b7af5ca9a65a03876a162e98d452beee Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 08:49:30 +0100
Subject: [PATCH 04/25] libclc: Use separate acquire and release fences in
work_group_barrier (#185190)
---
.../synchronization/clc_work_group_barrier.h | 2 +-
.../synchronization/clc_work_group_barrier.cl | 20 ++++++++++++++++---
.../synchronization/clc_work_group_barrier.cl | 3 +--
.../synchronization/work_group_barrier.cl | 3 +--
4 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/libclc/clc/include/clc/synchronization/clc_work_group_barrier.h b/libclc/clc/include/clc/synchronization/clc_work_group_barrier.h
index 34745bd47c068..e98dc38e1b0b3 100644
--- a/libclc/clc/include/clc/synchronization/clc_work_group_barrier.h
+++ b/libclc/clc/include/clc/synchronization/clc_work_group_barrier.h
@@ -13,7 +13,7 @@
#include <clc/mem_fence/clc_mem_semantic.h>
_CLC_OVERLOAD _CLC_DECL void
-__clc_work_group_barrier(int memory_scope, int memory_order,
+__clc_work_group_barrier(int memory_scope,
__CLC_MemorySemantics memory_semantics);
#endif // __CLC_SYNCHRONIZATION_CLC_WORK_GROUP_BARRIER_H__
diff --git a/libclc/clc/lib/amdgcn/synchronization/clc_work_group_barrier.cl b/libclc/clc/lib/amdgcn/synchronization/clc_work_group_barrier.cl
index 034e6e7bd8ed4..67b3d9b2f308b 100644
--- a/libclc/clc/lib/amdgcn/synchronization/clc_work_group_barrier.cl
+++ b/libclc/clc/lib/amdgcn/synchronization/clc_work_group_barrier.cl
@@ -10,8 +10,22 @@
#include <clc/synchronization/clc_work_group_barrier.h>
_CLC_OVERLOAD _CLC_DEF void
-__clc_work_group_barrier(int memory_scope, int memory_order,
+__clc_work_group_barrier(int memory_scope,
__CLC_MemorySemantics memory_semantics) {
- __clc_mem_fence(memory_scope, memory_order, memory_semantics);
- __builtin_amdgcn_s_barrier();
+ if (memory_semantics == 0) {
+ __builtin_amdgcn_s_barrier();
+ } else {
+ int memory_order_before =
+ memory_semantics & (__CLC_MEMORY_GLOBAL | __CLC_MEMORY_LOCAL)
+ ? __ATOMIC_SEQ_CST
+ : __ATOMIC_RELEASE;
+ int memory_order_after =
+ memory_semantics & (__CLC_MEMORY_GLOBAL | __CLC_MEMORY_LOCAL)
+ ? __ATOMIC_SEQ_CST
+ : __ATOMIC_ACQUIRE;
+
+ __clc_mem_fence(memory_scope, memory_order_before, memory_semantics);
+ __builtin_amdgcn_s_barrier();
+ __clc_mem_fence(memory_scope, memory_order_after, memory_semantics);
+ }
}
diff --git a/libclc/clc/lib/ptx-nvidiacl/synchronization/clc_work_group_barrier.cl b/libclc/clc/lib/ptx-nvidiacl/synchronization/clc_work_group_barrier.cl
index 3afc88ca50b15..35b381052367d 100644
--- a/libclc/clc/lib/ptx-nvidiacl/synchronization/clc_work_group_barrier.cl
+++ b/libclc/clc/lib/ptx-nvidiacl/synchronization/clc_work_group_barrier.cl
@@ -9,10 +9,9 @@
#include <clc/synchronization/clc_work_group_barrier.h>
_CLC_OVERLOAD _CLC_DEF void
-__clc_work_group_barrier(int memory_scope, int memory_order,
+__clc_work_group_barrier(int memory_scope,
__CLC_MemorySemantics memory_semantics) {
(void)memory_scope;
- (void)memory_order;
(void)memory_semantics;
__syncthreads();
}
diff --git a/libclc/opencl/lib/generic/synchronization/work_group_barrier.cl b/libclc/opencl/lib/generic/synchronization/work_group_barrier.cl
index 14de313c4f582..595c7f8cd95a6 100644
--- a/libclc/opencl/lib/generic/synchronization/work_group_barrier.cl
+++ b/libclc/opencl/lib/generic/synchronization/work_group_barrier.cl
@@ -12,9 +12,8 @@
_CLC_DEF _CLC_OVERLOAD void work_group_barrier(cl_mem_fence_flags flags,
memory_scope scope) {
- int memory_order = __ATOMIC_SEQ_CST;
__CLC_MemorySemantics memory_semantics = __opencl_get_memory_semantics(flags);
- __clc_work_group_barrier(__opencl_get_clang_memory_scope(scope), memory_order,
+ __clc_work_group_barrier(__opencl_get_clang_memory_scope(scope),
memory_semantics);
}
>From 5785568ca12c78b33c535a67f096c16c7f19dfca Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 09:07:52 +0100
Subject: [PATCH 05/25] libclc: Remove target opencl copies of mem_fence
(#185207)
---
libclc/opencl/lib/amdgcn/SOURCES | 1 -
libclc/opencl/lib/generic/SOURCES | 1 +
.../{amdgcn => generic}/mem_fence/fence.cl | 0
libclc/opencl/lib/ptx-nvidiacl/SOURCES | 1 -
.../lib/ptx-nvidiacl/mem_fence/fence.cl | 31 -------------------
5 files changed, 1 insertion(+), 33 deletions(-)
rename libclc/opencl/lib/{amdgcn => generic}/mem_fence/fence.cl (100%)
delete mode 100644 libclc/opencl/lib/ptx-nvidiacl/SOURCES
delete mode 100644 libclc/opencl/lib/ptx-nvidiacl/mem_fence/fence.cl
diff --git a/libclc/opencl/lib/amdgcn/SOURCES b/libclc/opencl/lib/amdgcn/SOURCES
index caf08495e8dae..8771edfc28f48 100644
--- a/libclc/opencl/lib/amdgcn/SOURCES
+++ b/libclc/opencl/lib/amdgcn/SOURCES
@@ -1,5 +1,4 @@
async/wait_group_events.cl
-mem_fence/fence.cl
printf/__printf_alloc.cl
subgroup/subgroup.cl
synchronization/sub_group_barrier.cl
diff --git a/libclc/opencl/lib/generic/SOURCES b/libclc/opencl/lib/generic/SOURCES
index 398e326ffe482..c29f79b82e882 100644
--- a/libclc/opencl/lib/generic/SOURCES
+++ b/libclc/opencl/lib/generic/SOURCES
@@ -175,6 +175,7 @@ math/tanh.cl
math/tanpi.cl
math/tgamma.cl
math/trunc.cl
+mem_fence/fence.cl
misc/shuffle.cl
misc/shuffle2.cl
relational/all.cl
diff --git a/libclc/opencl/lib/amdgcn/mem_fence/fence.cl b/libclc/opencl/lib/generic/mem_fence/fence.cl
similarity index 100%
rename from libclc/opencl/lib/amdgcn/mem_fence/fence.cl
rename to libclc/opencl/lib/generic/mem_fence/fence.cl
diff --git a/libclc/opencl/lib/ptx-nvidiacl/SOURCES b/libclc/opencl/lib/ptx-nvidiacl/SOURCES
deleted file mode 100644
index f20917346a3bc..0000000000000
--- a/libclc/opencl/lib/ptx-nvidiacl/SOURCES
+++ /dev/null
@@ -1 +0,0 @@
-mem_fence/fence.cl
diff --git a/libclc/opencl/lib/ptx-nvidiacl/mem_fence/fence.cl b/libclc/opencl/lib/ptx-nvidiacl/mem_fence/fence.cl
deleted file mode 100644
index 38fb15c2c1de8..0000000000000
--- a/libclc/opencl/lib/ptx-nvidiacl/mem_fence/fence.cl
+++ /dev/null
@@ -1,31 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <clc/mem_fence/clc_mem_fence.h>
-#include <clc/opencl/synchronization/utils.h>
-
-_CLC_DEF _CLC_OVERLOAD void mem_fence(cl_mem_fence_flags flags) {
- int memory_scope = __MEMORY_SCOPE_WRKGRP;
- int memory_order = __ATOMIC_ACQ_REL;
- __CLC_MemorySemantics memory_semantics = __opencl_get_memory_semantics(flags);
- __clc_mem_fence(memory_scope, memory_order, memory_semantics);
-}
-
-_CLC_DEF _CLC_OVERLOAD void read_mem_fence(cl_mem_fence_flags flags) {
- int memory_scope = __MEMORY_SCOPE_WRKGRP;
- int memory_order = __ATOMIC_ACQUIRE;
- __CLC_MemorySemantics memory_semantics = __opencl_get_memory_semantics(flags);
- __clc_mem_fence(memory_scope, memory_order, memory_semantics);
-}
-
-_CLC_DEF _CLC_OVERLOAD void write_mem_fence(cl_mem_fence_flags flags) {
- int memory_scope = __MEMORY_SCOPE_WRKGRP;
- int memory_order = __ATOMIC_RELEASE;
- __CLC_MemorySemantics memory_semantics = __opencl_get_memory_semantics(flags);
- __clc_mem_fence(memory_scope, memory_order, memory_semantics);
-}
>From 4c9d448e058d2a6371634a27608ef3fff01e817d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 09:09:40 +0100
Subject: [PATCH 06/25] libclc: Move sub_group_barrier to clc (#185208)
---
.../synchronization/clc_sub_group_barrier.h | 21 +++++++++++++++++++
libclc/clc/lib/amdgcn/SOURCES | 1 +
.../synchronization/clc_sub_group_barrier.cl | 18 ++++++++++++++++
libclc/clc/lib/generic/SOURCES | 1 +
.../lib/generic/subgroup/sub_group_barrier.cl | 14 +++++++++++++
libclc/opencl/lib/amdgcn/SOURCES | 1 -
libclc/opencl/lib/generic/SOURCES | 1 +
.../synchronization/sub_group_barrier.cl | 14 +++++++------
8 files changed, 64 insertions(+), 7 deletions(-)
create mode 100644 libclc/clc/include/clc/synchronization/clc_sub_group_barrier.h
create mode 100644 libclc/clc/lib/amdgcn/synchronization/clc_sub_group_barrier.cl
create mode 100644 libclc/clc/lib/generic/subgroup/sub_group_barrier.cl
rename libclc/opencl/lib/{amdgcn => generic}/synchronization/sub_group_barrier.cl (56%)
diff --git a/libclc/clc/include/clc/synchronization/clc_sub_group_barrier.h b/libclc/clc/include/clc/synchronization/clc_sub_group_barrier.h
new file mode 100644
index 0000000000000..58ca3d5b5a028
--- /dev/null
+++ b/libclc/clc/include/clc/synchronization/clc_sub_group_barrier.h
@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_SUBGROUP_CLC_SUB_GROUP_BARRIER_H__
+#define __CLC_SUBGROUP_CLC_SUB_GROUP_BARRIER_H__
+
+#include "clc/internal/clc.h"
+#include "clc/mem_fence/clc_mem_semantic.h"
+
+_CLC_DECL _CLC_OVERLOAD void
+__clc_sub_group_barrier(__CLC_MemorySemantics memory_semantics,
+ int memory_scope);
+_CLC_DECL _CLC_OVERLOAD void
+__clc_sub_group_barrier(__CLC_MemorySemantics memory_semantics);
+
+#endif // __CLC_SUBGROUP_CLC_SUB_GROUP_BARRIER_H__
diff --git a/libclc/clc/lib/amdgcn/SOURCES b/libclc/clc/lib/amdgcn/SOURCES
index 28ce5f1943825..a280461b1664a 100644
--- a/libclc/clc/lib/amdgcn/SOURCES
+++ b/libclc/clc/lib/amdgcn/SOURCES
@@ -2,6 +2,7 @@ address_space/qualifier.cl
math/clc_ldexp.cl
mem_fence/clc_mem_fence.cl
subgroup/sub_group_broadcast.cl
+synchronization/clc_sub_group_barrier.cl
synchronization/clc_work_group_barrier.cl
workitem/clc_get_enqueued_local_size.cl
workitem/clc_get_global_offset.cl
diff --git a/libclc/clc/lib/amdgcn/synchronization/clc_sub_group_barrier.cl b/libclc/clc/lib/amdgcn/synchronization/clc_sub_group_barrier.cl
new file mode 100644
index 0000000000000..ae26b1125f5c0
--- /dev/null
+++ b/libclc/clc/lib/amdgcn/synchronization/clc_sub_group_barrier.cl
@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/mem_fence/clc_mem_fence.h"
+#include "clc/synchronization/clc_sub_group_barrier.h"
+
+_CLC_DEF _CLC_OVERLOAD void
+__clc_sub_group_barrier(__CLC_MemorySemantics memory_semantics, int scope) {
+ __builtin_amdgcn_wave_barrier();
+
+ if (memory_semantics)
+ __clc_mem_fence(scope, __ATOMIC_ACQ_REL, memory_semantics);
+}
diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES
index 926efe9c6f188..b7ca616299556 100644
--- a/libclc/clc/lib/generic/SOURCES
+++ b/libclc/clc/lib/generic/SOURCES
@@ -176,6 +176,7 @@ shared/clc_min.cl
shared/clc_qualifier.cl
shared/clc_vload.cl
shared/clc_vstore.cl
+subgroup/sub_group_barrier.cl
workitem/clc_get_global_id.cl
workitem/clc_get_global_linear_id.cl
workitem/clc_get_local_linear_id.cl
diff --git a/libclc/clc/lib/generic/subgroup/sub_group_barrier.cl b/libclc/clc/lib/generic/subgroup/sub_group_barrier.cl
new file mode 100644
index 0000000000000..45f0a19b9ee39
--- /dev/null
+++ b/libclc/clc/lib/generic/subgroup/sub_group_barrier.cl
@@ -0,0 +1,14 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/synchronization/clc_sub_group_barrier.h"
+
+_CLC_DEF _CLC_OVERLOAD void
+__clc_sub_group_barrier(__CLC_MemorySemantics memory_semantics) {
+ __clc_sub_group_barrier(memory_semantics, __MEMORY_SCOPE_WVFRNT);
+}
diff --git a/libclc/opencl/lib/amdgcn/SOURCES b/libclc/opencl/lib/amdgcn/SOURCES
index 8771edfc28f48..7010953d28100 100644
--- a/libclc/opencl/lib/amdgcn/SOURCES
+++ b/libclc/opencl/lib/amdgcn/SOURCES
@@ -1,4 +1,3 @@
async/wait_group_events.cl
printf/__printf_alloc.cl
subgroup/subgroup.cl
-synchronization/sub_group_barrier.cl
diff --git a/libclc/opencl/lib/generic/SOURCES b/libclc/opencl/lib/generic/SOURCES
index c29f79b82e882..8e2df4a3e910a 100644
--- a/libclc/opencl/lib/generic/SOURCES
+++ b/libclc/opencl/lib/generic/SOURCES
@@ -202,6 +202,7 @@ shared/min.cl
shared/vload.cl
shared/vstore.cl
subgroup/sub_group_broadcast.cl
+synchronization/sub_group_barrier.cl
synchronization/work_group_barrier.cl
workitem/get_enqueued_local_size.cl
workitem/get_global_id.cl
diff --git a/libclc/opencl/lib/amdgcn/synchronization/sub_group_barrier.cl b/libclc/opencl/lib/generic/synchronization/sub_group_barrier.cl
similarity index 56%
rename from libclc/opencl/lib/amdgcn/synchronization/sub_group_barrier.cl
rename to libclc/opencl/lib/generic/synchronization/sub_group_barrier.cl
index 2b57d86294ef5..e03eb01cd767f 100644
--- a/libclc/opencl/lib/amdgcn/synchronization/sub_group_barrier.cl
+++ b/libclc/opencl/lib/generic/synchronization/sub_group_barrier.cl
@@ -6,16 +6,18 @@
//
//===----------------------------------------------------------------------===//
-#include <clc/opencl/opencl-base.h>
+#include "clc/opencl/synchronization/utils.h"
+#include "clc/opencl/utils.h"
+#include "clc/synchronization/clc_sub_group_barrier.h"
_CLC_DEF _CLC_OVERLOAD void sub_group_barrier(cl_mem_fence_flags flags,
memory_scope scope) {
- __builtin_amdgcn_wave_barrier();
-
- if (flags)
- atomic_work_item_fence(flags, memory_order_acq_rel, scope);
+ __CLC_MemorySemantics memory_semantics = __opencl_get_memory_semantics(flags);
+ __clc_sub_group_barrier(memory_semantics,
+ __opencl_get_clang_memory_scope(scope));
}
_CLC_DEF _CLC_OVERLOAD void sub_group_barrier(cl_mem_fence_flags flags) {
- sub_group_barrier(flags, memory_scope_sub_group);
+ __CLC_MemorySemantics memory_semantics = __opencl_get_memory_semantics(flags);
+ __clc_sub_group_barrier(memory_semantics);
}
>From 327f16f7fbdc33d0d0c8995d76aa31249718fcef Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 09:12:30 +0100
Subject: [PATCH 07/25] libclc: Rename sub_group_broadcast header (#185212)
The other clc headers have the clc prefix, so add it.
---
.../{sub_group_broadcast.h => clc_sub_group_broadcast.h} | 0
libclc/clc/lib/amdgcn/subgroup/sub_group_broadcast.cl | 2 +-
libclc/opencl/lib/generic/subgroup/sub_group_broadcast.cl | 2 +-
3 files changed, 2 insertions(+), 2 deletions(-)
rename libclc/clc/include/clc/subgroup/{sub_group_broadcast.h => clc_sub_group_broadcast.h} (100%)
diff --git a/libclc/clc/include/clc/subgroup/sub_group_broadcast.h b/libclc/clc/include/clc/subgroup/clc_sub_group_broadcast.h
similarity index 100%
rename from libclc/clc/include/clc/subgroup/sub_group_broadcast.h
rename to libclc/clc/include/clc/subgroup/clc_sub_group_broadcast.h
diff --git a/libclc/clc/lib/amdgcn/subgroup/sub_group_broadcast.cl b/libclc/clc/lib/amdgcn/subgroup/sub_group_broadcast.cl
index 458d14b7ae523..d6dc95b574910 100644
--- a/libclc/clc/lib/amdgcn/subgroup/sub_group_broadcast.cl
+++ b/libclc/clc/lib/amdgcn/subgroup/sub_group_broadcast.cl
@@ -6,7 +6,7 @@
//
//===----------------------------------------------------------------------===//
-#include "clc/subgroup/sub_group_broadcast.h"
+#include "clc/subgroup/clc_sub_group_broadcast.h"
_CLC_DEF _CLC_OVERLOAD _CLC_CONST char
__clc_sub_group_broadcast(char x, uint sub_group_local_id) {
diff --git a/libclc/opencl/lib/generic/subgroup/sub_group_broadcast.cl b/libclc/opencl/lib/generic/subgroup/sub_group_broadcast.cl
index a2e4fae6795cc..9401653d089bc 100644
--- a/libclc/opencl/lib/generic/subgroup/sub_group_broadcast.cl
+++ b/libclc/opencl/lib/generic/subgroup/sub_group_broadcast.cl
@@ -6,7 +6,7 @@
//
//===----------------------------------------------------------------------===//
-#include "clc/subgroup/sub_group_broadcast.h"
+#include "clc/subgroup/clc_sub_group_broadcast.h"
#define __CLC_BODY <sub_group_broadcast.inc>
#include <clc/integer/gentype.inc>
>From 78c6ebd2cdfc0fc810448d0fc23a18bc39b363cf Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 09:24:33 +0100
Subject: [PATCH 08/25] libclc: Move subgroup functions into clc (#185220)
It turns out there was a generic implementation of the id and sizes.
The practice of splitting every single function into its own file is
kind of a pain here, so introduce a utility header for amdgpu.
---
libclc/clc/include/clc/amdgpu/amdgpu_utils.h | 27 +++++++++
.../clc/include/clc/subgroup/clc_subgroup.h | 23 +++++++
libclc/clc/lib/amdgcn/SOURCES | 4 ++
libclc/clc/lib/amdgcn/subgroup/subgroup.cl | 28 +++++++++
.../workitem/clc_get_max_sub_group_size.cl | 7 ++-
.../amdgcn/workitem/clc_get_num_sub_groups.cl | 16 +++++
.../amdgcn/workitem/clc_get_sub_group_id.cl | 15 +++++
.../amdgcn/workitem/clc_get_sub_group_size.cl | 18 ++++++
libclc/opencl/lib/amdgcn/SOURCES | 1 -
libclc/opencl/lib/amdgcn/subgroup/subgroup.cl | 60 -------------------
libclc/opencl/lib/generic/SOURCES | 1 +
.../opencl/lib/generic/subgroup/subgroup.cl | 41 +++++++++++++
12 files changed, 178 insertions(+), 63 deletions(-)
create mode 100644 libclc/clc/include/clc/amdgpu/amdgpu_utils.h
create mode 100644 libclc/clc/include/clc/subgroup/clc_subgroup.h
create mode 100644 libclc/clc/lib/amdgcn/subgroup/subgroup.cl
create mode 100644 libclc/clc/lib/amdgcn/workitem/clc_get_num_sub_groups.cl
create mode 100644 libclc/clc/lib/amdgcn/workitem/clc_get_sub_group_id.cl
create mode 100644 libclc/clc/lib/amdgcn/workitem/clc_get_sub_group_size.cl
delete mode 100644 libclc/opencl/lib/amdgcn/subgroup/subgroup.cl
create mode 100644 libclc/opencl/lib/generic/subgroup/subgroup.cl
diff --git a/libclc/clc/include/clc/amdgpu/amdgpu_utils.h b/libclc/clc/include/clc/amdgpu/amdgpu_utils.h
new file mode 100644
index 0000000000000..40c5d770c7bde
--- /dev/null
+++ b/libclc/clc/include/clc/amdgpu/amdgpu_utils.h
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/integer/clc_mul24.h"
+#include "clc/workitem/clc_get_enqueued_local_size.h"
+#include "clc/workitem/clc_get_local_size.h"
+
+static inline uint __clc_amdgpu_workgroup_size() {
+ return __clc_mul24((uint)__clc_get_local_size(2),
+ __clc_mul24((uint)__clc_get_local_size(1),
+ (uint)__clc_get_local_size(0)));
+}
+
+static inline uint __clc_amdgpu_enqueued_workgroup_size() {
+ return __clc_mul24((uint)__clc_get_enqueued_local_size(2),
+ __clc_mul24((uint)__clc_get_enqueued_local_size(1),
+ (uint)__clc_get_enqueued_local_size(0)));
+}
+
+static inline uint __clc_amdgpu_wavesize_log2() {
+ return __builtin_amdgcn_wavefrontsize() == 64 ? 6 : 5;
+}
diff --git a/libclc/clc/include/clc/subgroup/clc_subgroup.h b/libclc/clc/include/clc/subgroup/clc_subgroup.h
new file mode 100644
index 0000000000000..f0a2a11d48445
--- /dev/null
+++ b/libclc/clc/include/clc/subgroup/clc_subgroup.h
@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_SUBGROUP_CLC_SUB_GROUP_SUBGROUP_H__
+#define __CLC_SUBGROUP_CLC_SUB_GROUP_SUBGROUP_H__
+
+#include "clc/internal/clc.h"
+
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST uint __clc_get_sub_group_size(void);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST uint __clc_get_max_sub_group_size(void);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST uint __clc_get_num_sub_groups(void);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST uint __clc_get_enqueued_num_sub_groups(void);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST uint __clc_get_sub_group_id(void);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST uint __clc_get_sub_group_local_id(void);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST int __clc_sub_group_all(int x);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST int __clc_sub_group_any(int x);
+
+#endif // __CLC_SUBGROUP_CLC_SUB_GROUP_SUBGROUP_H__
diff --git a/libclc/clc/lib/amdgcn/SOURCES b/libclc/clc/lib/amdgcn/SOURCES
index a280461b1664a..a0b6c168b207e 100644
--- a/libclc/clc/lib/amdgcn/SOURCES
+++ b/libclc/clc/lib/amdgcn/SOURCES
@@ -1,6 +1,7 @@
address_space/qualifier.cl
math/clc_ldexp.cl
mem_fence/clc_mem_fence.cl
+subgroup/subgroup.cl
subgroup/sub_group_broadcast.cl
synchronization/clc_sub_group_barrier.cl
synchronization/clc_work_group_barrier.cl
@@ -12,4 +13,7 @@ workitem/clc_get_local_id.cl
workitem/clc_get_local_size.cl
workitem/clc_get_max_sub_group_size.cl
workitem/clc_get_num_groups.cl
+workitem/clc_get_num_sub_groups.cl
+workitem/clc_get_sub_group_id.cl
+workitem/clc_get_sub_group_size.cl
workitem/clc_get_work_dim.cl
diff --git a/libclc/clc/lib/amdgcn/subgroup/subgroup.cl b/libclc/clc/lib/amdgcn/subgroup/subgroup.cl
new file mode 100644
index 0000000000000..71f4abc42e895
--- /dev/null
+++ b/libclc/clc/lib/amdgcn/subgroup/subgroup.cl
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/amdgpu/amdgpu_utils.h"
+#include "clc/subgroup/clc_subgroup.h"
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint __clc_get_enqueued_num_sub_groups(void) {
+ return (__clc_amdgpu_enqueued_workgroup_size() +
+ __builtin_amdgcn_wavefrontsize() - 1) >>
+ __clc_amdgpu_wavesize_log2();
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint __clc_get_sub_group_local_id(void) {
+ return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u));
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST int __clc_sub_group_all(int x) {
+ return __builtin_amdgcn_ballot_w64(x) == __builtin_amdgcn_read_exec();
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST int __clc_sub_group_any(int x) {
+ return __builtin_amdgcn_ballot_w64(x) != 0;
+}
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl
index cc56f8d9c325d..7df7f21d9098f 100644
--- a/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl
@@ -6,8 +6,11 @@
//
//===----------------------------------------------------------------------===//
-#include <clc/workitem/clc_get_max_sub_group_size.h>
+#include "clc/amdgpu/amdgpu_utils.h"
+#include "clc/shared/clc_min.h"
+#include "clc/workitem/clc_get_max_sub_group_size.h"
_CLC_OVERLOAD _CLC_DEF uint __clc_get_max_sub_group_size() {
- return __builtin_amdgcn_wavefrontsize();
+ return __clc_min(__builtin_amdgcn_wavefrontsize(),
+ __clc_amdgpu_enqueued_workgroup_size());
}
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_num_sub_groups.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_num_sub_groups.cl
new file mode 100644
index 0000000000000..cb71ef282466b
--- /dev/null
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_num_sub_groups.cl
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/amdgpu/amdgpu_utils.h"
+#include "clc/subgroup/clc_subgroup.h"
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint __clc_get_num_sub_groups(void) {
+ uint group_size = __clc_amdgpu_workgroup_size();
+ return (group_size + __builtin_amdgcn_wavefrontsize() - 1) >>
+ __clc_amdgpu_wavesize_log2();
+}
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_sub_group_id.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_sub_group_id.cl
new file mode 100644
index 0000000000000..ba3baf98bda14
--- /dev/null
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_sub_group_id.cl
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/amdgpu/amdgpu_utils.h"
+#include "clc/workitem/clc_get_local_linear_id.h"
+#include "clc/workitem/clc_get_sub_group_id.h"
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint __clc_get_sub_group_id(void) {
+ return (uint)__clc_get_local_linear_id() >> __clc_amdgpu_wavesize_log2();
+}
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_sub_group_size.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_sub_group_size.cl
new file mode 100644
index 0000000000000..77c9f8e91d8ee
--- /dev/null
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_sub_group_size.cl
@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/amdgpu/amdgpu_utils.h"
+#include "clc/shared/clc_min.h"
+#include "clc/workitem/clc_get_local_linear_id.h"
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint __clc_get_sub_group_size(void) {
+ uint wavesize = __builtin_amdgcn_wavefrontsize();
+ uint lid = (uint)__clc_get_local_linear_id();
+ return __clc_min(wavesize,
+ __clc_amdgpu_workgroup_size() - (lid & ~(wavesize - 1)));
+}
diff --git a/libclc/opencl/lib/amdgcn/SOURCES b/libclc/opencl/lib/amdgcn/SOURCES
index 7010953d28100..78877425504d6 100644
--- a/libclc/opencl/lib/amdgcn/SOURCES
+++ b/libclc/opencl/lib/amdgcn/SOURCES
@@ -1,3 +1,2 @@
async/wait_group_events.cl
printf/__printf_alloc.cl
-subgroup/subgroup.cl
diff --git a/libclc/opencl/lib/amdgcn/subgroup/subgroup.cl b/libclc/opencl/lib/amdgcn/subgroup/subgroup.cl
deleted file mode 100644
index d67d84e763b4f..0000000000000
--- a/libclc/opencl/lib/amdgcn/subgroup/subgroup.cl
+++ /dev/null
@@ -1,60 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <clc/opencl/opencl-base.h>
-
-static uint wavesize_log2() {
- return __builtin_amdgcn_wavefrontsize() == 64 ? 6 : 5;
-}
-
-static uint workgroup_size() {
- return mul24((uint)get_local_size(2),
- mul24((uint)get_local_size(1), (uint)get_local_size(0)));
-}
-
-static uint enqueued_workgroup_size() {
- return mul24((uint)get_enqueued_local_size(2),
- mul24((uint)get_enqueued_local_size(1),
- (uint)get_enqueued_local_size(0)));
-}
-
-_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_sub_group_size(void) {
- uint wavesize = __builtin_amdgcn_wavefrontsize();
- uint lid = (uint)get_local_linear_id();
- return min(wavesize, workgroup_size() - (lid & ~(wavesize - 1)));
-}
-
-_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_max_sub_group_size(void) {
- return min(__builtin_amdgcn_wavefrontsize(), enqueued_workgroup_size());
-}
-
-_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_num_sub_groups(void) {
- return (workgroup_size() + __builtin_amdgcn_wavefrontsize() - 1) >>
- wavesize_log2();
-}
-
-_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_enqueued_num_sub_groups(void) {
- return (enqueued_workgroup_size() + __builtin_amdgcn_wavefrontsize() - 1) >>
- wavesize_log2();
-}
-
-_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_sub_group_id(void) {
- return (uint)get_local_linear_id() >> wavesize_log2();
-}
-
-_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_sub_group_local_id(void) {
- return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u));
-}
-
-_CLC_DEF _CLC_OVERLOAD _CLC_CONST int sub_group_all(int x) {
- return __builtin_amdgcn_ballot_w64(x) == __builtin_amdgcn_read_exec();
-}
-
-_CLC_DEF _CLC_OVERLOAD _CLC_CONST int sub_group_any(int x) {
- return __builtin_amdgcn_ballot_w64(x) != 0;
-}
diff --git a/libclc/opencl/lib/generic/SOURCES b/libclc/opencl/lib/generic/SOURCES
index 8e2df4a3e910a..f735c66548c30 100644
--- a/libclc/opencl/lib/generic/SOURCES
+++ b/libclc/opencl/lib/generic/SOURCES
@@ -201,6 +201,7 @@ shared/max.cl
shared/min.cl
shared/vload.cl
shared/vstore.cl
+subgroup/subgroup.cl
subgroup/sub_group_broadcast.cl
synchronization/sub_group_barrier.cl
synchronization/work_group_barrier.cl
diff --git a/libclc/opencl/lib/generic/subgroup/subgroup.cl b/libclc/opencl/lib/generic/subgroup/subgroup.cl
new file mode 100644
index 0000000000000..fd552ada4afaf
--- /dev/null
+++ b/libclc/opencl/lib/generic/subgroup/subgroup.cl
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/subgroup/clc_subgroup.h"
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_sub_group_size(void) {
+ return __clc_get_sub_group_size();
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_max_sub_group_size(void) {
+ return __clc_get_max_sub_group_size();
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_num_sub_groups(void) {
+ return __clc_get_num_sub_groups();
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_enqueued_num_sub_groups(void) {
+ return __clc_get_enqueued_num_sub_groups();
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_sub_group_id(void) {
+ return __clc_get_sub_group_id();
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST uint get_sub_group_local_id(void) {
+ return __clc_get_sub_group_local_id();
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST int sub_group_all(int x) {
+ return __clc_sub_group_all(x);
+}
+
+_CLC_DEF _CLC_OVERLOAD _CLC_CONST int sub_group_any(int x) {
+ return __clc_sub_group_any(x);
+}
>From 68bb8a0ded61c11b7bc7f91f8d4c7a9bbb04b7d0 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 09:41:20 +0100
Subject: [PATCH 09/25] libclc: Add work_group_any/work_group_all
implementation (#185260)
---
.../clc/collective/clc_work_group_any_all.h | 17 ++++++
libclc/clc/lib/generic/SOURCES | 1 +
.../collective/clc_work_group_any_all.cl | 58 +++++++++++++++++++
libclc/opencl/lib/generic/SOURCES | 1 +
.../generic/collective/work_group_any_all.cl | 17 ++++++
5 files changed, 94 insertions(+)
create mode 100644 libclc/clc/include/clc/collective/clc_work_group_any_all.h
create mode 100644 libclc/clc/lib/generic/collective/clc_work_group_any_all.cl
create mode 100644 libclc/opencl/lib/generic/collective/work_group_any_all.cl
diff --git a/libclc/clc/include/clc/collective/clc_work_group_any_all.h b/libclc/clc/include/clc/collective/clc_work_group_any_all.h
new file mode 100644
index 0000000000000..40c8c515f5998
--- /dev/null
+++ b/libclc/clc/include/clc/collective/clc_work_group_any_all.h
@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_COLLECTIVE_CLC_WORK_GROUP_ANY_ALL_H__
+#define __CLC_COLLECTIVE_CLC_WORK_GROUP_ANY_ALL_H__
+
+#include "clc/internal/clc.h"
+
+_CLC_OVERLOAD _CLC_DECL int __clc_work_group_any(int predicate);
+_CLC_OVERLOAD _CLC_DECL int __clc_work_group_all(int predicate);
+
+#endif // __CLC_COLLECTIVE_CLC_WORK_GROUP_ANY_ALL_H__
diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES
index b7ca616299556..0e9166a2b249e 100644
--- a/libclc/clc/lib/generic/SOURCES
+++ b/libclc/clc/lib/generic/SOURCES
@@ -14,6 +14,7 @@ atomic/clc_atomic_flag_test_and_set.cl
atomic/clc_atomic_inc.cl
atomic/clc_atomic_load.cl
atomic/clc_atomic_store.cl
+collective/clc_work_group_any_all.cl
common/clc_degrees.cl
common/clc_radians.cl
common/clc_sign.cl
diff --git a/libclc/clc/lib/generic/collective/clc_work_group_any_all.cl b/libclc/clc/lib/generic/collective/clc_work_group_any_all.cl
new file mode 100644
index 0000000000000..4c79ef1f73eba
--- /dev/null
+++ b/libclc/clc/lib/generic/collective/clc_work_group_any_all.cl
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/atomic/clc_atomic_fetch_and.h"
+#include "clc/atomic/clc_atomic_fetch_or.h"
+#include "clc/atomic/clc_atomic_load.h"
+#include "clc/atomic/clc_atomic_store.h"
+#include "clc/collective/clc_work_group_any_all.h"
+#include "clc/subgroup/clc_subgroup.h"
+#include "clc/synchronization/clc_work_group_barrier.h"
+
+#pragma OPENCL EXTENSION __cl_clang_function_scope_local_variables : enable
+
+static int work_group_any_all_impl(int predicate, bool is_all) {
+ __local uint scratch;
+
+ uint n = __clc_get_num_sub_groups();
+ int a =
+ is_all ? __clc_sub_group_all(predicate) : __clc_sub_group_any(predicate);
+ if (n == 1)
+ return a;
+
+ uint l = __clc_get_sub_group_local_id();
+ uint i = __clc_get_sub_group_id();
+
+ if ((i == 0) & (l == 0))
+ __clc_atomic_store(&scratch, a, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP);
+
+ __clc_work_group_barrier(__MEMORY_SCOPE_WRKGRP, __CLC_MEMORY_LOCAL);
+
+ if ((i != 0) & (l == 0)) {
+ if (is_all)
+ __clc_atomic_fetch_and(&scratch, a, __ATOMIC_RELAXED,
+ __MEMORY_SCOPE_WRKGRP);
+ else
+ __clc_atomic_fetch_or(&scratch, a, __ATOMIC_RELAXED,
+ __MEMORY_SCOPE_WRKGRP);
+ }
+
+ __clc_work_group_barrier(__MEMORY_SCOPE_WRKGRP, __CLC_MEMORY_LOCAL);
+ a = __clc_atomic_load(&scratch, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP);
+ __clc_work_group_barrier(__MEMORY_SCOPE_WRKGRP, __CLC_MEMORY_LOCAL);
+
+ return a;
+}
+
+_CLC_OVERLOAD _CLC_DEF int __clc_work_group_all(int predicate) {
+ return work_group_any_all_impl(predicate, true);
+}
+
+_CLC_OVERLOAD _CLC_DEF int __clc_work_group_any(int predicate) {
+ return work_group_any_all_impl(predicate, false);
+}
diff --git a/libclc/opencl/lib/generic/SOURCES b/libclc/opencl/lib/generic/SOURCES
index f735c66548c30..5dc92c27f9d74 100644
--- a/libclc/opencl/lib/generic/SOURCES
+++ b/libclc/opencl/lib/generic/SOURCES
@@ -42,6 +42,7 @@ atomic/atom_sub.cl
atomic/atom_xchg.cl
atomic/atom_xor.cl
atomic/atomic_work_item_fence.cl
+collective/work_group_any_all.cl
common/degrees.cl
common/mix.cl
common/radians.cl
diff --git a/libclc/opencl/lib/generic/collective/work_group_any_all.cl b/libclc/opencl/lib/generic/collective/work_group_any_all.cl
new file mode 100644
index 0000000000000..7d1391deaa2e3
--- /dev/null
+++ b/libclc/opencl/lib/generic/collective/work_group_any_all.cl
@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/collective/clc_work_group_any_all.h"
+
+_CLC_OVERLOAD _CLC_DEF int work_group_all(int predicate) {
+ return __clc_work_group_all(predicate);
+}
+
+_CLC_OVERLOAD _CLC_DEF int work_group_any(int predicate) {
+ return __clc_work_group_any(predicate);
+}
>From abb3b981a37a04e56cea6439961d6bb3ee058f67 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 09:54:30 +0100
Subject: [PATCH 10/25] libclc: Add work_group_broadcast (#185261)
---
.../clc/collective/clc_work_group_broadcast.h | 20 ++++++++
.../collective/clc_work_group_broadcast.inc | 16 ++++++
libclc/clc/lib/generic/SOURCES | 1 +
.../collective/clc_work_group_broadcast.cl | 23 +++++++++
.../collective/clc_work_group_broadcast.inc | 51 +++++++++++++++++++
libclc/opencl/lib/generic/SOURCES | 1 +
.../collective/work_group_broadcast.cl | 15 ++++++
.../collective/work_group_broadcast.inc | 29 +++++++++++
8 files changed, 156 insertions(+)
create mode 100644 libclc/clc/include/clc/collective/clc_work_group_broadcast.h
create mode 100644 libclc/clc/include/clc/collective/clc_work_group_broadcast.inc
create mode 100644 libclc/clc/lib/generic/collective/clc_work_group_broadcast.cl
create mode 100644 libclc/clc/lib/generic/collective/clc_work_group_broadcast.inc
create mode 100644 libclc/opencl/lib/generic/collective/work_group_broadcast.cl
create mode 100644 libclc/opencl/lib/generic/collective/work_group_broadcast.inc
diff --git a/libclc/clc/include/clc/collective/clc_work_group_broadcast.h b/libclc/clc/include/clc/collective/clc_work_group_broadcast.h
new file mode 100644
index 0000000000000..b8a5a1dff1f31
--- /dev/null
+++ b/libclc/clc/include/clc/collective/clc_work_group_broadcast.h
@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_COLLECTIVE_CLC_WORK_GROUP_BROADCAST_H__
+#define __CLC_COLLECTIVE_CLC_WORK_GROUP_BROADCAST_H__
+
+#include "clc/internal/clc.h"
+
+#define __CLC_BODY <clc/collective/clc_work_group_broadcast.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/collective/clc_work_group_broadcast.inc>
+#include <clc/math/gentype.inc>
+
+#endif // __CLC_COLLECTIVE_CLC_WORK_GROUP_BROADCAST_H__
diff --git a/libclc/clc/include/clc/collective/clc_work_group_broadcast.inc b/libclc/clc/include/clc/collective/clc_work_group_broadcast.inc
new file mode 100644
index 0000000000000..64633fa06a047
--- /dev/null
+++ b/libclc/clc/include/clc/collective/clc_work_group_broadcast.inc
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
+__clc_work_group_broadcast(__CLC_GENTYPE a, size_t local_id_x);
+
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_work_group_broadcast(
+ __CLC_GENTYPE a, size_t local_id_x, size_t local_id_y);
+
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_work_group_broadcast(
+ __CLC_GENTYPE a, size_t local_id_x, size_t local_id_y, size_t local_id_z);
diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES
index 0e9166a2b249e..05ea37869bbf8 100644
--- a/libclc/clc/lib/generic/SOURCES
+++ b/libclc/clc/lib/generic/SOURCES
@@ -15,6 +15,7 @@ atomic/clc_atomic_inc.cl
atomic/clc_atomic_load.cl
atomic/clc_atomic_store.cl
collective/clc_work_group_any_all.cl
+collective/clc_work_group_broadcast.cl
common/clc_degrees.cl
common/clc_radians.cl
common/clc_sign.cl
diff --git a/libclc/clc/lib/generic/collective/clc_work_group_broadcast.cl b/libclc/clc/lib/generic/collective/clc_work_group_broadcast.cl
new file mode 100644
index 0000000000000..d54bda65773f8
--- /dev/null
+++ b/libclc/clc/lib/generic/collective/clc_work_group_broadcast.cl
@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/atomic/clc_atomic_load.h"
+#include "clc/atomic/clc_atomic_store.h"
+#include "clc/collective/clc_work_group_broadcast.h"
+#include "clc/subgroup/clc_sub_group_broadcast.h"
+#include "clc/subgroup/clc_subgroup.h"
+#include "clc/synchronization/clc_work_group_barrier.h"
+#include "clc/workitem/clc_get_local_id.h"
+
+#pragma OPENCL EXTENSION __cl_clang_function_scope_local_variables : enable
+
+#define __CLC_BODY <clc_work_group_broadcast.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc_work_group_broadcast.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/collective/clc_work_group_broadcast.inc b/libclc/clc/lib/generic/collective/clc_work_group_broadcast.inc
new file mode 100644
index 0000000000000..a431560cf10d0
--- /dev/null
+++ b/libclc/clc/lib/generic/collective/clc_work_group_broadcast.inc
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __CLC_SCALAR
+
+_CLC_OVERLOAD
+static __CLC_GENTYPE __clc_work_group_broadcast_impl(__CLC_GENTYPE a,
+ bool is_leader) {
+ __local __CLC_GENTYPE scratch;
+ if (is_leader) {
+ __scoped_atomic_store_n(&scratch, a, __ATOMIC_RELAXED,
+ __MEMORY_SCOPE_WRKGRP);
+ }
+
+ __clc_work_group_barrier(__MEMORY_SCOPE_WRKGRP, __CLC_MEMORY_LOCAL);
+ __CLC_GENTYPE result =
+ __scoped_atomic_load_n(&scratch, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP);
+ __clc_work_group_barrier(__MEMORY_SCOPE_WRKGRP, __CLC_MEMORY_LOCAL);
+ return result;
+}
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE
+__clc_work_group_broadcast(__CLC_GENTYPE a, size_t local_id_x) {
+ if (__clc_get_num_sub_groups() == 1)
+ return __clc_sub_group_broadcast(a, local_id_x);
+
+ bool is_leader = __clc_get_local_id(0) == local_id_x;
+ return __clc_work_group_broadcast_impl(a, is_leader);
+}
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_work_group_broadcast(
+ __CLC_GENTYPE a, size_t local_id_x, size_t local_id_y) {
+ bool is_leader = __clc_get_local_id(0) == local_id_x &&
+ __clc_get_local_id(1) == local_id_y;
+ return __clc_work_group_broadcast_impl(a, is_leader);
+}
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_work_group_broadcast(
+ __CLC_GENTYPE a, size_t local_id_x, size_t local_id_y, size_t local_id_z) {
+ bool is_leader = __clc_get_local_id(0) == local_id_x &&
+ __clc_get_local_id(1) == local_id_y &&
+ __clc_get_local_id(2) == local_id_z;
+ return __clc_work_group_broadcast_impl(a, is_leader);
+}
+
+#endif // __CLC_SCALAR
diff --git a/libclc/opencl/lib/generic/SOURCES b/libclc/opencl/lib/generic/SOURCES
index 5dc92c27f9d74..0afbb67ebe1e4 100644
--- a/libclc/opencl/lib/generic/SOURCES
+++ b/libclc/opencl/lib/generic/SOURCES
@@ -43,6 +43,7 @@ atomic/atom_xchg.cl
atomic/atom_xor.cl
atomic/atomic_work_item_fence.cl
collective/work_group_any_all.cl
+collective/work_group_broadcast.cl
common/degrees.cl
common/mix.cl
common/radians.cl
diff --git a/libclc/opencl/lib/generic/collective/work_group_broadcast.cl b/libclc/opencl/lib/generic/collective/work_group_broadcast.cl
new file mode 100644
index 0000000000000..fa4e8fbc26bb3
--- /dev/null
+++ b/libclc/opencl/lib/generic/collective/work_group_broadcast.cl
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clc/collective/clc_work_group_broadcast.h"
+
+#define __CLC_BODY <work_group_broadcast.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <work_group_broadcast.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/collective/work_group_broadcast.inc b/libclc/opencl/lib/generic/collective/work_group_broadcast.inc
new file mode 100644
index 0000000000000..7b9695a7181c3
--- /dev/null
+++ b/libclc/opencl/lib/generic/collective/work_group_broadcast.inc
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__CLC_SCALAR) && (defined(__CLC_FPSIZE) || __CLC_GENSIZE >= 32)
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE work_group_broadcast(__CLC_GENTYPE a,
+ size_t local_id_x) {
+ return __clc_work_group_broadcast(a, local_id_x);
+}
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE work_group_broadcast(__CLC_GENTYPE a,
+ size_t local_id_x,
+ size_t local_id_y) {
+ return __clc_work_group_broadcast(a, local_id_x, local_id_y);
+}
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE work_group_broadcast(__CLC_GENTYPE a,
+ size_t local_id_x,
+ size_t local_id_y,
+ size_t local_id_z) {
+ return __clc_work_group_broadcast(a, local_id_x, local_id_y, local_id_z);
+}
+
+#endif
>From a2fd44acf3115bc5cb024814296b19df6ab34287 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 10:03:51 +0100
Subject: [PATCH 11/25] libclc: Fix wait_group_events build for targets without
generic (#185264)
---
libclc/opencl/lib/amdgcn/async/wait_group_events.cl | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/libclc/opencl/lib/amdgcn/async/wait_group_events.cl b/libclc/opencl/lib/amdgcn/async/wait_group_events.cl
index fac33f73af1c4..f1c9925847de5 100644
--- a/libclc/opencl/lib/amdgcn/async/wait_group_events.cl
+++ b/libclc/opencl/lib/amdgcn/async/wait_group_events.cl
@@ -15,9 +15,13 @@ _CLC_DEF _CLC_OVERLOAD void wait_group_events(int n, __private event_t *evs) {
memory_scope_work_group);
}
-_CLC_DEF _CLC_OVERLOAD void wait_group_events(int n, __generic event_t *evs) {
+#if _CLC_GENERIC_AS_SUPPORTED
+
+_CLC_DEF _CLC_OVERLOAD void wait_group_events(int n, event_t *evs) {
(void)n;
(void)evs;
work_group_barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE,
memory_scope_work_group);
}
+
+#endif
>From 7752b2efa2c796c69d1db95cdf89eb2b8194fcf8 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 10:16:04 +0100
Subject: [PATCH 12/25] libclc: Add uintptr_t overloads of atomic_fetch_add and
sub (#185263)
This is a special case because the pointee type is unsigned, but the
input value is signed. Directly use the opencl builtins, because these
work correctly without any ugly casting required, and it's not worth
putting a wrapper in clc.
---
.../lib/generic/atomic/atomic_fetch_add.cl | 28 +++++++++++++++++++
.../lib/generic/atomic/atomic_fetch_sub.cl | 28 +++++++++++++++++++
2 files changed, 56 insertions(+)
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl
index 428ac19830f7f..8e431ace788e0 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl
@@ -17,3 +17,31 @@
#define __CLC_BODY <atomic_def.inc>
#include <clc/math/gentype.inc>
+
+#if defined(__opencl_c_atomic_order_seq_cst) && \
+ defined(__opencl_c_atomic_scope_device)
+
+_CLC_OVERLOAD _CLC_DEF uintptr_t
+atomic_fetch_add(volatile __local atomic_uintptr_t *p, ptrdiff_t v) {
+ return __opencl_atomic_fetch_add(p, v, memory_order_seq_cst,
+ memory_scope_device);
+}
+
+_CLC_OVERLOAD _CLC_DEF uintptr_t
+atomic_fetch_add(volatile __global atomic_uintptr_t *p, ptrdiff_t v) {
+ return __opencl_atomic_fetch_add(p, v, memory_order_seq_cst,
+ memory_scope_device);
+}
+
+#if _CLC_GENERIC_AS_SUPPORTED
+
+_CLC_OVERLOAD _CLC_DEF uintptr_t atomic_fetch_add(volatile atomic_uintptr_t *p,
+ ptrdiff_t v) {
+ return __opencl_atomic_fetch_add(p, v, memory_order_seq_cst,
+ memory_scope_device);
+}
+
+#endif // _CLC_GENERIC_AS_SUPPORTED
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+ // defined(__opencl_c_atomic_scope_device)
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl
index 474aca6658014..e7bd865de7a58 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl
@@ -17,3 +17,31 @@
#define __CLC_BODY <atomic_def.inc>
#include <clc/math/gentype.inc>
+
+#if defined(__opencl_c_atomic_order_seq_cst) && \
+ defined(__opencl_c_atomic_scope_device)
+
+_CLC_OVERLOAD _CLC_DEF uintptr_t
+atomic_fetch_sub(volatile __local atomic_uintptr_t *p, ptrdiff_t v) {
+ return __opencl_atomic_fetch_sub(p, v, memory_order_seq_cst,
+ memory_scope_device);
+}
+
+_CLC_OVERLOAD _CLC_DEF uintptr_t
+atomic_fetch_sub(volatile __global atomic_uintptr_t *p, ptrdiff_t v) {
+ return __opencl_atomic_fetch_sub(p, v, memory_order_seq_cst,
+ memory_scope_device);
+}
+
+#if _CLC_GENERIC_AS_SUPPORTED
+
+_CLC_OVERLOAD _CLC_DEF uintptr_t atomic_fetch_sub(volatile atomic_uintptr_t *p,
+ ptrdiff_t v) {
+ return __opencl_atomic_fetch_sub(p, v, memory_order_seq_cst,
+ memory_scope_device);
+}
+
+#endif // _CLC_GENERIC_AS_SUPPORTED
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+ // defined(__opencl_c_atomic_scope_device)
>From 03347e0834884a6ed795a5eccc0057e705919698 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 8 Mar 2026 10:26:25 +0100
Subject: [PATCH 13/25] libclc: Correctly declare __clc_get_max_sub_group_size
as taking no arguments (#185265)
---
libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h | 2 +-
libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h b/libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h
index c8ea2a1fead12..0e3b67f64b809 100644
--- a/libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h
+++ b/libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h
@@ -11,6 +11,6 @@
#include <clc/internal/clc.h>
-_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint __clc_get_max_sub_group_size();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint __clc_get_max_sub_group_size(void);
#endif // __CLC_WORKITEM_CLC_GET_MAX_SUB_GROUP_SIZE_H__
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl
index 7df7f21d9098f..5eb0166135663 100644
--- a/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl
@@ -10,7 +10,7 @@
#include "clc/shared/clc_min.h"
#include "clc/workitem/clc_get_max_sub_group_size.h"
-_CLC_OVERLOAD _CLC_DEF uint __clc_get_max_sub_group_size() {
+_CLC_OVERLOAD _CLC_DEF uint __clc_get_max_sub_group_size(void) {
return __clc_min(__builtin_amdgcn_wavefrontsize(),
__clc_amdgpu_enqueued_workgroup_size());
}
>From 327f1adef8df6afc07f6c88cfa380c97399af3dc Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Sun, 8 Mar 2026 09:38:42 +0000
Subject: [PATCH 14/25] [AArch64] Ensure FPR128 callee-save stack offsets are
aligned (#184314)
This was benign for Linux targets (as when dividing by the scale the
offset would be correctly truncated), so only resulted in failures with
`-DLLVM_ENABLE_ASSERTIONS=On`. On Windows, this was a miscompile as the
lack of alignment would result in the FPR128 callee-save getting
assigned to the same offset as the previous GPR.
Fixes: #183708
---
.../Target/AArch64/AArch64FrameLowering.cpp | 20 +++++++---
.../CodeGen/AArch64/framelayout-fpr128-csr.ll | 33 ++++++++++++++++
.../AArch64/framelayout-fpr128-spill.mir | 38 +++++++++++++++++++
3 files changed, 86 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll
create mode 100644 llvm/test/CodeGen/AArch64/framelayout-fpr128-spill.mir
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index e524c98984ee7..6790b80086136 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1730,6 +1730,12 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
Register LastReg = 0;
bool HasCSHazardPadding = AFI->hasStackHazardSlotIndex() && !SplitPPRs;
+ auto AlignOffset = [StackFillDir](int Offset, int Align) {
+ if (StackFillDir < 0)
+ return alignDown(Offset, Align);
+ return alignTo(Offset, Align);
+ };
+
// When iterating backwards, the loop condition relies on unsigned wraparound.
for (unsigned i = FirstReg; i < Count; i += RegInc) {
RegPairInfo RPI;
@@ -1844,11 +1850,15 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
- // Realign the scalable offset if necessary. This is relevant when
- // spilling predicates on Windows.
- if (RPI.isScalable() && ScalableByteOffset % Scale != 0) {
- ScalableByteOffset = alignTo(ScalableByteOffset, Scale);
- }
+ // Realign the scalable offset if necessary. This is relevant when spilling
+ // predicates on Windows.
+ if (RPI.isScalable() && ScalableByteOffset % Scale != 0)
+ ScalableByteOffset = AlignOffset(ScalableByteOffset, Scale);
+
+ // Realign the fixed offset if necessary. This is relevant when spilling Q
+ // registers after spilling an odd amount of X registers.
+ if (!RPI.isScalable() && ByteOffset % Scale != 0)
+ ByteOffset = AlignOffset(ByteOffset, Scale);
int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
assert(OffsetPre % Scale == 0);
diff --git a/llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll b/llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll
new file mode 100644
index 0000000000000..09bfbc5d4c82b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-windows-msvc < %s | FileCheck %s --check-prefix=CHECK-WINDOWS
+
+; The purpose of this test is to verify q8 is assigned a 16-byte aligned offset
+; after the x10 is assigned an offset. The CSR (on Linux) are assigned offsets
+; in the order GPRs then FPRs. The stack size of this function is 48
+; (alignTo((16 + 8 * 3), 16)), so after x8 is given the offset 24, q8 originally
+; would be assigned offset 8, which is not 16-byte aligned.
+define preserve_allcc void @d(ptr %ptr) nounwind {
+; CHECK-LABEL: d:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: str q8, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT: str x10, [sp, #24] // 8-byte Spill
+; CHECK-NEXT: //APP
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ldr x10, [sp, #24] // 8-byte Reload
+; CHECK-NEXT: ldr q8, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+;
+; CHECK-WINDOWS-LABEL: d:
+; CHECK-WINDOWS: // %bb.0: // %entry
+; CHECK-WINDOWS-NEXT: str x10, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-WINDOWS-NEXT: str q8, [sp, #16] // 16-byte Spill
+; CHECK-WINDOWS-NEXT: //APP
+; CHECK-WINDOWS-NEXT: //NO_APP
+; CHECK-WINDOWS-NEXT: ldr q8, [sp, #16] // 16-byte Reload
+; CHECK-WINDOWS-NEXT: ldr x10, [sp], #32 // 8-byte Folded Reload
+; CHECK-WINDOWS-NEXT: ret
+entry:
+ tail call void asm sideeffect "", "~{x10},~{q8}"()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/framelayout-fpr128-spill.mir b/llvm/test/CodeGen/AArch64/framelayout-fpr128-spill.mir
new file mode 100644
index 0000000000000..a6236bd917129
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/framelayout-fpr128-spill.mir
@@ -0,0 +1,38 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64-linux-gnu -run-pass=prologepilog %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64-windows-msvc -run-pass=prologepilog %s -o - | FileCheck %s --check-prefix=CHECK-WINDOWS
+
+--- |
+ ; Tests Q8 is assigned a 16-byte aligned offset after X10 is assigned an offset.
+ define preserve_allcc void @test_fpr128_spill_alignment() nounwind { entry: unreachable }
+...
+---
+name: test_fpr128_spill_alignment
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_fpr128_spill_alignment
+ ; CHECK: liveins: $q8, $x10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: early-clobber $sp = frame-setup STRQpre killed $q8, $sp, -32 :: (store (s128) into %stack.1)
+ ; CHECK-NEXT: frame-setup STRXui killed $x10, $sp, 3 :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: $q8 = IMPLICIT_DEF
+ ; CHECK-NEXT: $x10 = IMPLICIT_DEF
+ ; CHECK-NEXT: $x10 = frame-destroy LDRXui $sp, 3 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: early-clobber $sp, $q8 = frame-destroy LDRQpost $sp, 32 :: (load (s128) from %stack.1)
+ ; CHECK-NEXT: RET_ReallyLR
+ ;
+ ; CHECK-WINDOWS-LABEL: name: test_fpr128_spill_alignment
+ ; CHECK-WINDOWS: liveins: $x10, $q8
+ ; CHECK-WINDOWS-NEXT: {{ $}}
+ ; CHECK-WINDOWS-NEXT: early-clobber $sp = frame-setup STRXpre killed $x10, $sp, -32 :: (store (s64) into %stack.1)
+ ; CHECK-WINDOWS-NEXT: frame-setup STRQui killed $q8, $sp, 1 :: (store (s128) into %stack.0)
+ ; CHECK-WINDOWS-NEXT: $q8 = IMPLICIT_DEF
+ ; CHECK-WINDOWS-NEXT: $x10 = IMPLICIT_DEF
+ ; CHECK-WINDOWS-NEXT: $q8 = frame-destroy LDRQui $sp, 1 :: (load (s128) from %stack.0)
+ ; CHECK-WINDOWS-NEXT: early-clobber $sp, $x10 = frame-destroy LDRXpost $sp, 32 :: (load (s64) from %stack.1)
+ ; CHECK-WINDOWS-NEXT: RET_ReallyLR
+ $q8 = IMPLICIT_DEF
+ $x10 = IMPLICIT_DEF
+ RET_ReallyLR
+...
>From 4c31b6f93c7d8499b93cd6d29b8874a62f2cfed0 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <macdue at dueutil.tech>
Date: Sun, 8 Mar 2026 10:31:57 +0000
Subject: [PATCH 15/25] Fix comment in FPR128 test (NFC)
---
llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll b/llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll
index 09bfbc5d4c82b..4cce7ec8a47cd 100644
--- a/llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll
+++ b/llvm/test/CodeGen/AArch64/framelayout-fpr128-csr.ll
@@ -3,9 +3,9 @@
; RUN: llc -verify-machineinstrs -mtriple=aarch64-windows-msvc < %s | FileCheck %s --check-prefix=CHECK-WINDOWS
; The purpose of this test is to verify q8 is assigned a 16-byte aligned offset
-; after the x10 is assigned an offset. The CSR (on Linux) are assigned offsets
-; in the order GPRs then FPRs. The stack size of this function is 48
-; (alignTo((16 + 8 * 3), 16)), so after x8 is given the offset 24, q8 originally
+; after the x10 is assigned an offset. The CSRs (on Linux) are assigned offsets
+; in the order GPRs then FPRs. The stack size of this function is 32
+; (alignTo((16 + 8), 16)), so after x8 is given the offset 24, q8 originally
; would be assigned offset 8, which is not 16-byte aligned.
define preserve_allcc void @d(ptr %ptr) nounwind {
; CHECK-LABEL: d:
>From b73ae4eea84d44731c1b1e9d4e7b7462b515aee3 Mon Sep 17 00:00:00 2001
From: Henrich Lauko <xlauko at mail.muni.cz>
Date: Sun, 8 Mar 2026 11:44:54 +0100
Subject: [PATCH 16/25] [CIR] Extract base classes for complex ops (#185262)
Extract CIR_ComplexPartOp for ComplexRealOp/ComplexImagOp,
CIR_ComplexPartPtrOp for ComplexRealPtrOp/ComplexImagPtrOp,
CIR_ComplexBinOp for ComplexAddOp/ComplexSubOp, and
CIR_ComplexRangeBinOp for ComplexMulOp/ComplexDivOp to
eliminate duplicated arguments, results, format, and traits.
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 182 +++++++------------
1 file changed, 64 insertions(+), 118 deletions(-)
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 79eef71229192..7f52be17d30f8 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -4964,24 +4964,10 @@ def CIR_ComplexCreateOp : CIR_Op<"complex.create", [Pure, SameTypeOperands]> {
}
//===----------------------------------------------------------------------===//
-// ComplexRealOp
+// ComplexRealOp and ComplexImagOp
//===----------------------------------------------------------------------===//
-def CIR_ComplexRealOp : CIR_Op<"complex.real", [Pure]> {
- let summary = "Extract the real part of a complex value";
- let description = [{
- `cir.complex.real` operation takes an operand of `!cir.complex`, `cir.int`,
- `!cir.bool` or `!cir.float`. If the operand is `!cir.complex`, the real
- part of it will be returned, otherwise the value returned unmodified.
-
- Example:
-
- ```mlir
- %real = cir.complex.real %complex : !cir.complex<!cir.float> -> !cir.float
- %real = cir.complex.real %scalar : !cir.float -> !cir.float
- ```
- }];
-
+class CIR_ComplexPartOp<string mnemonic> : CIR_Op<mnemonic, [Pure]> {
let results = (outs CIR_AnyIntOrBoolOrFloatType:$result);
let arguments = (ins CIR_AnyComplexOrIntOrBoolOrFloatType:$operand);
@@ -4994,16 +4980,28 @@ def CIR_ComplexRealOp : CIR_Op<"complex.real", [Pure]> {
let hasFolder = 1;
}
-//===----------------------------------------------------------------------===//
-// ComplexImagOp
-//===----------------------------------------------------------------------===//
+def CIR_ComplexRealOp : CIR_ComplexPartOp<"complex.real"> {
+ let summary = "Extract the real part of a complex value";
+ let description = [{
+ `cir.complex.real` operation takes an operand of `!cir.complex`, `cir.int`,
+ `!cir.bool` or `!cir.float`. If the operand is `!cir.complex`, the real
+ part of it will be returned, otherwise the value returned unmodified.
+
+ Example:
-def CIR_ComplexImagOp : CIR_Op<"complex.imag", [Pure]> {
+ ```mlir
+ %real = cir.complex.real %complex : !cir.complex<!cir.float> -> !cir.float
+ %real = cir.complex.real %scalar : !cir.float -> !cir.float
+ ```
+ }];
+}
+
+def CIR_ComplexImagOp : CIR_ComplexPartOp<"complex.imag"> {
let summary = "Extract the imaginary part of a complex value";
let description = [{
`cir.complex.imag` operation takes an operand of `!cir.complex`, `!cir.int`
- `!cir.bool` or `!cir.float`. If the operand is `!cir.complex`, the imag
- part of it will be returned, otherwise a zero value will be returned.
+ `!cir.bool` or `!cir.float`. If the operand is `!cir.complex`, the imag
+ part of it will be returned, otherwise a zero value will be returned.
Example:
@@ -5012,24 +5010,25 @@ def CIR_ComplexImagOp : CIR_Op<"complex.imag", [Pure]> {
%imag = cir.complex.imag %scalar : !cir.float -> !cir.float
```
}];
+}
- let results = (outs CIR_AnyIntOrBoolOrFloatType:$result);
- let arguments = (ins CIR_AnyComplexOrIntOrBoolOrFloatType:$operand);
+//===----------------------------------------------------------------------===//
+// ComplexRealPtrOp and ComplexImagPtrOp
+//===----------------------------------------------------------------------===//
+
+class CIR_ComplexPartPtrOp<string mnemonic> : CIR_Op<mnemonic, [Pure]> {
+ let arguments = (ins CIR_PtrToComplexType:$operand);
+ let results = (outs CIR_PtrToIntOrFloatType:$result);
let assemblyFormat = [{
- $operand `:` qualified(type($operand)) `->` qualified(type($result))
- attr-dict
+ $operand `:`
+ qualified(type($operand)) `->` qualified(type($result)) attr-dict
}];
let hasVerifier = 1;
- let hasFolder = 1;
}
-//===----------------------------------------------------------------------===//
-// ComplexRealPtrOp
-//===----------------------------------------------------------------------===//
-
-def CIR_ComplexRealPtrOp : CIR_Op<"complex.real_ptr", [Pure]> {
+def CIR_ComplexRealPtrOp : CIR_ComplexPartPtrOp<"complex.real_ptr"> {
let summary = "Derive a pointer to the real part of a complex value";
let description = [{
`cir.complex.real_ptr` operation takes a pointer operand that points to a
@@ -5043,23 +5042,9 @@ def CIR_ComplexRealPtrOp : CIR_Op<"complex.real_ptr", [Pure]> {
-> !cir.ptr<!cir.double>
```
}];
-
- let results = (outs CIR_PtrToIntOrFloatType:$result);
- let arguments = (ins CIR_PtrToComplexType:$operand);
-
- let assemblyFormat = [{
- $operand `:`
- qualified(type($operand)) `->` qualified(type($result)) attr-dict
- }];
-
- let hasVerifier = 1;
}
-//===----------------------------------------------------------------------===//
-// ComplexImagPtrOp
-//===----------------------------------------------------------------------===//
-
-def CIR_ComplexImagPtrOp : CIR_Op<"complex.imag_ptr", [Pure]> {
+def CIR_ComplexImagPtrOp : CIR_ComplexPartPtrOp<"complex.imag_ptr"> {
let summary = "Derive a pointer to the imaginary part of a complex value";
let description = [{
`cir.complex.imag_ptr` operation takes a pointer operand that points to a
@@ -5073,25 +5058,23 @@ def CIR_ComplexImagPtrOp : CIR_Op<"complex.imag_ptr", [Pure]> {
-> !cir.ptr<!cir.double>
```
}];
+}
- let arguments = (ins CIR_PtrToComplexType:$operand);
- let results = (outs CIR_PtrToIntOrFloatType:$result);
+//===----------------------------------------------------------------------===//
+// ComplexAddOp and ComplexSubOp
+//===----------------------------------------------------------------------===//
+
+class CIR_ComplexBinOp<string mnemonic>
+ : CIR_Op<mnemonic, [Pure, SameOperandsAndResultType]> {
+ let arguments = (ins CIR_ComplexType:$lhs, CIR_ComplexType:$rhs);
+ let results = (outs CIR_ComplexType:$result);
let assemblyFormat = [{
- $operand `:`
- qualified(type($operand)) `->` qualified(type($result)) attr-dict
+ $lhs `,` $rhs `:` qualified(type($result)) attr-dict
}];
-
- let hasVerifier = 1;
}
-//===----------------------------------------------------------------------===//
-// ComplexAddOp
-//===----------------------------------------------------------------------===//
-
-def CIR_ComplexAddOp : CIR_Op<"complex.add", [
- Pure, SameOperandsAndResultType
-]> {
+def CIR_ComplexAddOp : CIR_ComplexBinOp<"complex.add"> {
let summary = "Complex addition";
let description = [{
The `cir.complex.add` operation takes two complex numbers and returns
@@ -5103,23 +5086,9 @@ def CIR_ComplexAddOp : CIR_Op<"complex.add", [
%2 = cir.complex.add %0, %1 : !cir.complex<!cir.float>
```
}];
-
- let arguments = (ins CIR_ComplexType:$lhs, CIR_ComplexType:$rhs);
-
- let results = (outs CIR_ComplexType:$result);
-
- let assemblyFormat = [{
- $lhs `,` $rhs `:` qualified(type($result)) attr-dict
- }];
}
-//===----------------------------------------------------------------------===//
-// ComplexSubOp
-//===----------------------------------------------------------------------===//
-
-def CIR_ComplexSubOp : CIR_Op<"complex.sub", [
- Pure, SameOperandsAndResultType
-]> {
+def CIR_ComplexSubOp : CIR_ComplexBinOp<"complex.sub"> {
let summary = "Complex subtraction";
let description = [{
The `cir.complex.sub` operation takes two complex numbers and returns
@@ -5131,18 +5100,10 @@ def CIR_ComplexSubOp : CIR_Op<"complex.sub", [
%2 = cir.complex.sub %0, %1 : !cir.complex<!cir.float>
```
}];
-
- let arguments = (ins CIR_ComplexType:$lhs, CIR_ComplexType:$rhs);
-
- let results = (outs CIR_ComplexType:$result);
-
- let assemblyFormat = [{
- $lhs `,` $rhs `:` qualified(type($result)) attr-dict
- }];
}
//===----------------------------------------------------------------------===//
-// ComplexMulOp & ComplexDivOp
+// ComplexMulOp and ComplexDivOp
//===----------------------------------------------------------------------===//
def CIR_ComplexRangeKind : CIR_I32EnumAttr<
@@ -5153,9 +5114,24 @@ def CIR_ComplexRangeKind : CIR_I32EnumAttr<
I32EnumAttrCase<"Basic", 3, "basic">,
]>;
-def CIR_ComplexMulOp : CIR_Op<"complex.mul", [
- Pure, SameOperandsAndResultType
-]> {
+class CIR_ComplexRangeBinOp<string mnemonic>
+ : CIR_Op<mnemonic, [Pure, SameOperandsAndResultType]> {
+ let arguments = (ins
+ CIR_ComplexType:$lhs,
+ CIR_ComplexType:$rhs,
+ CIR_ComplexRangeKind:$range
+ );
+
+ let results = (outs CIR_ComplexType:$result);
+
+ let assemblyFormat = [{
+ $lhs `,` $rhs `range` `(` $range `)` `:` qualified(type($result)) attr-dict
+ }];
+
+ let hasLLVMLowering = false;
+}
+
+def CIR_ComplexMulOp : CIR_ComplexRangeBinOp<"complex.mul"> {
let summary = "Complex multiplication";
let description = [{
The `cir.complex.mul` operation takes two complex numbers and returns
@@ -5176,25 +5152,9 @@ def CIR_ComplexMulOp : CIR_Op<"complex.mul", [
%2 = cir.complex.mul %0, %1 range(full) : !cir.complex<!cir.float>
```
}];
-
- let arguments = (ins
- CIR_ComplexType:$lhs,
- CIR_ComplexType:$rhs,
- CIR_ComplexRangeKind:$range
- );
-
- let results = (outs CIR_ComplexType:$result);
-
- let assemblyFormat = [{
- $lhs `,` $rhs `range` `(` $range `)` `:` qualified(type($result)) attr-dict
- }];
-
- let hasLLVMLowering = false;
}
-def CIR_ComplexDivOp : CIR_Op<"complex.div", [
- Pure, SameOperandsAndResultType
-]> {
+def CIR_ComplexDivOp : CIR_ComplexRangeBinOp<"complex.div"> {
let summary = "Complex division";
let description = [{
The `cir.complex.div` operation takes two complex numbers and returns
@@ -5220,20 +5180,6 @@ def CIR_ComplexDivOp : CIR_Op<"complex.div", [
%2 = cir.complex.div %0, %1 range(full) : !cir.complex<!cir.float>
```
}];
-
- let arguments = (ins
- CIR_ComplexType:$lhs,
- CIR_ComplexType:$rhs,
- CIR_ComplexRangeKind:$range
- );
-
- let results = (outs CIR_ComplexType:$result);
-
- let assemblyFormat = [{
- $lhs `,` $rhs `range` `(` $range `)` `:` qualified(type($result)) attr-dict
- }];
-
- let hasLLVMLowering = false;
}
//===----------------------------------------------------------------------===//
>From 6aa115bba55054b0dc81ebfc049e8c7a29e614b2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 8 Mar 2026 11:13:39 +0000
Subject: [PATCH 17/25] Reapply "[VPlan] Remove manual region removal when
simplifying for VF and UF. (#181252)"
This reverts commit d7e037c8383e66e5c07897f144f6d8ef47258682.
Recommit with a small fix to properly handle ordered reductions when
connecting the epilogue.
Original message:
Replace manual region dissolution code in
simplifyBranchConditionForVFAndUF with using general
removeBranchOnConst. simplifyBranchConditionForVFAndUF now just creates
a (BranchOnCond true) or updates BranchOnTwoConds.
The loop then gets automatically removed by running removeBranchOnConst.
This removes a bunch of special logic to handle header phi replacements
and CFG updates. With the new code, there's no restriction on what kind
of header phi recipes the loop contains.
Note that VPEVLBasedIVRecipe needs to be marked as readnone. This is
technically unrelated, but I could not find an independent test that
would be impacted.
The code to deal with epilogue resume values now needs updating, because
we may simplify a reduction directly to the start value.
PR: https://github.com/llvm/llvm-project/pull/181252
---
.../Transforms/Vectorize/LoopVectorize.cpp | 49 ++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 93 ++----
.../Transforms/Vectorize/VPlanTransforms.h | 5 +
.../AArch64/clamped-trip-count.ll | 40 +--
.../AArch64/force-target-instruction-cost.ll | 8 +-
.../AArch64/low_trip_count_predicates.ll | 18 +-
.../AArch64/mul-simplification.ll | 10 +-
.../LoopVectorize/AArch64/optsize_minsize.ll | 209 +++++--------
.../replicating-load-store-costs-apple.ll | 33 +-
.../AArch64/replicating-load-store-costs.ll | 31 +-
.../LoopVectorize/RISCV/low-trip-count.ll | 14 +-
...ctor-loop-backedge-elimination-with-evl.ll | 15 +-
.../LoopVectorize/X86/cost-model.ll | 38 +--
.../epilog-vectorization-ordered-reduction.ll | 132 ++++----
.../LoopVectorize/X86/load-deref-pred.ll | 119 +++----
.../epilog-vectorization-reductions.ll | 196 +++++++++++-
.../LoopVectorize/iv-select-cmp-decreasing.ll | 292 ++++++------------
...eref-pred-poison-ub-ops-feeding-pointer.ll | 179 ++++-------
.../reduction-minmax-users-and-predicated.ll | 58 ++--
.../single-early-exit-cond-poison.ll | 8 +-
...or-loop-backedge-elimination-early-exit.ll | 95 +++---
.../vector-loop-backedge-elimination.ll | 266 ++++++++++++++++
23 files changed, 996 insertions(+), 914 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb4eef5a41c09..420b111bafc72 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7370,21 +7370,45 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
vputils::findRecipe(BackedgeVal, IsaPred<VPReductionPHIRecipe>));
if (!EpiRedHeaderPhi) {
match(BackedgeVal,
- VPlanPatternMatch::m_Select(VPlanPatternMatch::m_VPValue(),
- VPlanPatternMatch::m_VPValue(BackedgeVal),
- VPlanPatternMatch::m_VPValue()));
- EpiRedHeaderPhi = cast<VPReductionPHIRecipe>(
+ m_Select(m_VPValue(), m_VPValue(BackedgeVal), m_VPValue()));
+ EpiRedHeaderPhi = cast_if_present<VPReductionPHIRecipe>(
vputils::findRecipe(BackedgeVal, IsaPred<VPReductionPHIRecipe>));
}
+ // Look through Broadcast or ReductionStartVector to get the underlying
+ // start value.
+ auto GetStartValue = [](VPValue *V) -> Value * {
+ VPValue *Start;
+ if (match(V, m_VPInstruction<VPInstruction::ReductionStartVector>(
+ m_VPValue(Start), m_VPValue(), m_VPValue())) ||
+ match(V, m_Broadcast(m_VPValue(Start))))
+ return Start->getUnderlyingValue();
+ return V->getUnderlyingValue();
+ };
+
Value *MainResumeValue;
- if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
- assert((VPI->getOpcode() == VPInstruction::Broadcast ||
- VPI->getOpcode() == VPInstruction::ReductionStartVector) &&
- "unexpected start recipe");
- MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
- } else
- MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
+ if (EpiRedHeaderPhi) {
+ MainResumeValue = GetStartValue(EpiRedHeaderPhi->getStartValue());
+ } else {
+ // The epilogue vector loop was dissolved (single-iteration). The
+ // reduction header phi was replaced by its start value. Look for a
+ // Broadcast or ReductionStartVector in BackedgeVal or its operands.
+ Value *FromOperand = nullptr;
+ if (auto *BackedgeR = BackedgeVal->getDefiningRecipe()) {
+ // For ordered (in-loop) reductions, BackedgeVal is a
+ // VPReductionRecipe whose chain operand is the start value.
+ if (auto *Red = dyn_cast<VPReductionRecipe>(BackedgeR)) {
+ FromOperand = GetStartValue(Red->getChainOp());
+ } else {
+ auto *It = find_if(BackedgeR->operands(), [&](VPValue *Op) {
+ return GetStartValue(Op) != Op->getUnderlyingValue();
+ });
+ if (It != BackedgeR->op_end())
+ FromOperand = GetStartValue(*It);
+ }
+ }
+ MainResumeValue = FromOperand ? FromOperand : GetStartValue(BackedgeVal);
+ }
if (EpiRedResult->getOpcode() == VPInstruction::ComputeAnyOfResult) {
[[maybe_unused]] Value *StartV =
EpiRedResult->getOperand(0)->getLiveInIRValue();
@@ -7466,6 +7490,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
VPlanTransforms::expandBranchOnTwoConds(BestVPlan);
// Convert loops with variable-length stepping after regions are dissolved.
VPlanTransforms::convertToVariableLengthStep(BestVPlan);
+ // Remove dead edges for single-iteration loops with BranchOnCond(true).
+ VPlanTransforms::removeBranchOnConst(BestVPlan);
VPlanTransforms::materializeBackedgeTakenCount(BestVPlan, VectorPH);
VPlanTransforms::materializeVectorTripCount(
BestVPlan, VectorPH, CM.foldTailByMasking(),
@@ -7473,6 +7499,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
VPlanTransforms::materializeFactors(BestVPlan, VectorPH, BestVF);
VPlanTransforms::cse(BestVPlan);
VPlanTransforms::simplifyRecipes(BestVPlan);
+ VPlanTransforms::simplifyKnownEVL(BestVPlan, BestVF, PSE);
// 0. Generate SCEV-dependent code in the entry, including TripCount, before
// making any changes to the CFG.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d149723e11fb6..e5d7ee1905136 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -128,6 +128,7 @@ bool VPRecipeBase::mayReadFromMemory() const {
return cast<VPWidenIntrinsicRecipe>(this)->mayReadFromMemory();
case VPBranchOnMaskSC:
case VPDerivedIVSC:
+ case VPCurrentIterationPHISC:
case VPFirstOrderRecurrencePHISC:
case VPReductionPHISC:
case VPPredInstPHISC:
@@ -165,6 +166,7 @@ bool VPRecipeBase::mayHaveSideEffects() const {
return cast<VPExpressionRecipe>(this)->mayHaveSideEffects();
case VPActiveLaneMaskPHISC:
case VPDerivedIVSC:
+ case VPCurrentIterationPHISC:
case VPFirstOrderRecurrencePHISC:
case VPReductionPHISC:
case VPPredInstPHISC:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index dc0edb178efbb..bd80cf91c7c85 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1525,12 +1525,6 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
return;
}
- if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Def)) {
- if (Phi->getOperand(0) == Phi->getOperand(1))
- Phi->replaceAllUsesWith(Phi->getOperand(0));
- return;
- }
-
// Simplify MaskedCond with no block mask to its single operand.
if (match(Def, m_VPInstruction<VPInstruction::MaskedCond>()) &&
!cast<VPInstruction>(Def)->isMasked())
@@ -1579,9 +1573,15 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
return;
}
- if (isa<VPPhi, VPWidenPHIRecipe>(Def)) {
- if (Def->getNumOperands() == 1)
+ if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Def)) {
+ if (Def->getNumOperands() == 1) {
Def->replaceAllUsesWith(Def->getOperand(0));
+ return;
+ }
+ if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Def)) {
+ if (all_equal(Phi->incoming_values()))
+ Phi->replaceAllUsesWith(Phi->getOperand(0));
+ }
return;
}
@@ -2153,72 +2153,16 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
return false;
}
- // The vector loop region only executes once. If possible, completely remove
- // the region, otherwise replace the terminator controlling the latch with
- // (BranchOnCond true).
- // TODO: VPWidenIntOrFpInductionRecipe is only partially supported; add
- // support for other non-canonical widen induction recipes (e.g.,
- // VPWidenPointerInductionRecipe).
- // TODO: fold branch-on-constant after dissolving region.
- auto *Header = cast<VPBasicBlock>(VectorRegion->getEntry());
- if (all_of(Header->phis(), [](VPRecipeBase &Phi) {
- if (auto *R = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi))
- return R->isCanonical();
- return isa<VPCanonicalIVPHIRecipe, VPCurrentIterationPHIRecipe,
- VPFirstOrderRecurrencePHIRecipe, VPPhi>(&Phi);
- })) {
- for (VPRecipeBase &HeaderR : make_early_inc_range(Header->phis())) {
- if (auto *R = dyn_cast<VPWidenIntOrFpInductionRecipe>(&HeaderR)) {
- VPBuilder Builder(Plan.getVectorPreheader());
- VPValue *StepV = Builder.createNaryOp(VPInstruction::StepVector, {},
- R->getScalarType());
- HeaderR.getVPSingleValue()->replaceAllUsesWith(StepV);
- HeaderR.eraseFromParent();
- continue;
- }
- auto *Phi = cast<VPPhiAccessors>(&HeaderR);
- HeaderR.getVPSingleValue()->replaceAllUsesWith(Phi->getIncomingValue(0));
- HeaderR.eraseFromParent();
- }
-
- VPBlockBase *Preheader = VectorRegion->getSinglePredecessor();
- SmallVector<VPBlockBase *> Exits = to_vector(VectorRegion->getSuccessors());
- VPBlockUtils::disconnectBlocks(Preheader, VectorRegion);
- for (VPBlockBase *Exit : Exits)
- VPBlockUtils::disconnectBlocks(VectorRegion, Exit);
-
- for (VPBlockBase *B : vp_depth_first_shallow(VectorRegion->getEntry()))
- B->setParent(nullptr);
-
- VPBlockUtils::connectBlocks(Preheader, Header);
-
- for (VPBlockBase *Exit : Exits)
- VPBlockUtils::connectBlocks(ExitingVPBB, Exit);
-
- // Replace terminating branch-on-two-conds with branch-on-cond to early
- // exit.
- if (Exits.size() != 1) {
- assert(match(Term, m_BranchOnTwoConds()) && Exits.size() == 2 &&
- "BranchOnTwoConds needs 2 remaining exits");
- VPBuilder(Term).createNaryOp(VPInstruction::BranchOnCond,
- Term->getOperand(0));
- }
- VPlanTransforms::simplifyRecipes(Plan);
- } else {
- // The vector region contains header phis for which we cannot remove the
- // loop region yet.
-
- // For BranchOnTwoConds, set the latch exit condition to true directly.
- if (match(Term, m_BranchOnTwoConds())) {
- Term->setOperand(1, Plan.getTrue());
- return true;
- }
-
- auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, {Plan.getTrue()},
- {}, {}, Term->getDebugLoc());
- ExitingVPBB->appendRecipe(BOC);
+ // The vector loop region only executes once. Convert terminator of the
+ // exiting block to exit in the first iteration.
+ if (match(Term, m_BranchOnTwoConds())) {
+ Term->setOperand(1, Plan.getTrue());
+ return true;
}
+ auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, Plan.getTrue(), {},
+ {}, Term->getDebugLoc());
+ ExitingVPBB->appendRecipe(BOC);
Term->eraseFromParent();
return true;
@@ -2226,8 +2170,8 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
/// From the definition of llvm.experimental.get.vector.length,
/// VPInstruction::ExplicitVectorLength(%AVL) = %AVL when %AVL <= VF.
-static bool simplifyKnownEVL(VPlan &Plan, ElementCount VF,
- PredicatedScalarEvolution &PSE) {
+bool VPlanTransforms::simplifyKnownEVL(VPlan &Plan, ElementCount VF,
+ PredicatedScalarEvolution &PSE) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry()))) {
for (VPRecipeBase &R : *VPBB) {
@@ -2262,7 +2206,6 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
bool MadeChange = tryToReplaceALMWithWideALM(Plan, BestVF, BestUF);
MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
- MadeChange |= simplifyKnownEVL(Plan, BestVF, PSE);
if (MadeChange) {
Plan.setVF(BestVF);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 2956659e5df8b..45e7be3169a52 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -239,6 +239,11 @@ struct VPlanTransforms {
unsigned BestUF,
PredicatedScalarEvolution &PSE);
+ /// Try to simplify VPInstruction::ExplicitVectorLength recipes when the AVL
+ /// is known to be <= VF, replacing them with the AVL directly.
+ static bool simplifyKnownEVL(VPlan &Plan, ElementCount VF,
+ PredicatedScalarEvolution &PSE);
+
/// Apply VPlan-to-VPlan optimizations to \p Plan, including induction recipe
/// optimizations, dead recipe removal, replicate region optimizations and
/// block merging.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
index e054c916de6e0..d54e8582676d6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
@@ -8,28 +8,17 @@ define void @clamped_tc_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range(1,1
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 8)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VAL]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
-; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP1]], i64 0
-; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP8]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw <vscale x 8 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw <vscale x 8 x i64> [[TMP8]], splat (i64 3)
; CHECK-NEXT: [[TMP11:%.*]] = lshr <vscale x 8 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
; CHECK-NEXT: [[TMP14:%.*]] = trunc <vscale x 8 x i64> [[TMP11]] to <vscale x 8 x i8>
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr align 1 [[NEXT_GEP]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 8)
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr align 1 [[DST]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]])
+; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.cond.cleanup:
@@ -68,28 +57,17 @@ define void @clamped_tc_max_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range
; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[SHR]] to i64
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VAL]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
-; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP1]], i64 0
-; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP8]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw <vscale x 8 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw <vscale x 8 x i64> [[TMP8]], splat (i64 3)
; CHECK-NEXT: [[TMP11:%.*]] = lshr <vscale x 8 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
; CHECK-NEXT: [[TMP14:%.*]] = trunc <vscale x 8 x i64> [[TMP11]] to <vscale x 8 x i8>
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr align 1 [[NEXT_GEP]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr align 1 [[DST]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]])
+; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.cond.cleanup.loopexit:
@@ -124,9 +102,3 @@ for.body: ; preds = %for.body.preheader,
for.cond.cleanup: ; preds = %for.body
ret void
}
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 0a62ac9804524..e4079d923e4af 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -13,11 +13,9 @@ define double @test_reduction_costs() {
; COMMON: [[VECTOR_PH]]:
; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; COMMON: [[VECTOR_BODY]]:
-; COMMON-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ]
-; COMMON-NEXT: [[VEC_PHI1:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
-; COMMON-NEXT: [[TMP0]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> splat (double 3.000000e+00))
-; COMMON-NEXT: [[TMP1]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI1]], <2 x double> splat (double 9.000000e+00))
-; COMMON-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMMON-NEXT: [[TMP0:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> splat (double 3.000000e+00))
+; COMMON-NEXT: [[TMP1:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> splat (double 9.000000e+00))
+; COMMON-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; COMMON: [[MIDDLE_BLOCK]]:
; COMMON-NEXT: br label %[[EXIT:.*]]
; COMMON: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
index c340cfc9ad6cc..6acda0d4b3294 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
@@ -464,13 +464,9 @@ define i32 @tc4(ptr noundef readonly captures(none) %tmp) vscale_range(1,16) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[ARRAYIDX1]], align 4
-; CHECK-NEXT: [[TMP3]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> zeroinitializer, [[WIDE_LOAD]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -509,7 +505,7 @@ define i32 @tc4_from_profile(ptr noundef readonly captures(none) %tmp, i64 %N) v
; CHECK-NEXT: [[ADD]] = add i32 [[SUM_0179]], [[TMP0]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]], !prof [[PROF9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]], !prof [[PROF8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ]
; CHECK-NEXT: ret i32 [[ADD_LCSSA]]
@@ -546,8 +542,7 @@ exit: ; preds = %for.body
; CHECK-VS1: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
; CHECK-VS1: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
; CHECK-VS1: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
-; CHECK-VS1: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK-VS1: [[PROF9]] = !{!"branch_weights", i32 10, i32 30}
+; CHECK-VS1: [[PROF8]] = !{!"branch_weights", i32 10, i32 30}
;.
; CHECK-VS2: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK-VS2: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -557,6 +552,5 @@ exit: ; preds = %for.body
; CHECK-VS2: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
; CHECK-VS2: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
; CHECK-VS2: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
-; CHECK-VS2: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK-VS2: [[PROF9]] = !{!"branch_weights", i32 10, i32 30}
+; CHECK-VS2: [[PROF8]] = !{!"branch_weights", i32 10, i32 30}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll b/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
index c4d06254a0d30..4b25bec39e51b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
@@ -51,17 +51,17 @@ define i32 @add_reduction_select_operand_constant_but_non_uniform() {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 42, i32 0, i32 0, i32 0>, %[[VECTOR_PH]] ], [ [[TMP2]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 42, i32 0, i32 0, i32 0>, %[[VECTOR_PH]] ], [ [[VEC_PHI]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[VEC_PHI1]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 64
; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[VEC_PHI1]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i32 [[TMP3]]
+; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
index 5b6979d6b1198..8b9c33ad5da76 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
@@ -196,165 +196,124 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
; DEFAULT-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT5]], <16 x i8> poison, <16 x i32> zeroinitializer
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
-; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE35:.*]] ]
-; DEFAULT-NEXT: [[VEC_IND:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE35]] ]
-; DEFAULT-NEXT: [[VEC_IND1:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT36:%.*]], %[[PRED_STORE_CONTINUE35]] ]
-; DEFAULT-NEXT: [[TMP0:%.*]] = icmp ule <16 x i8> [[VEC_IND]], splat (i8 14)
-; DEFAULT-NEXT: [[TMP1:%.*]] = mul <16 x i8> [[BROADCAST_SPLAT]], [[VEC_IND1]]
-; DEFAULT-NEXT: [[TMP2:%.*]] = lshr <16 x i8> [[VEC_IND1]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP3:%.*]] = mul <16 x i8> [[TMP2]], [[BROADCAST_SPLAT4]]
+; DEFAULT-NEXT: [[TMP1:%.*]] = mul <16 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>
+; DEFAULT-NEXT: [[TMP3:%.*]] = mul <16 x i8> <i8 0, i8 0, i8 1, i8 1, i8 2, i8 2, i8 3, i8 3, i8 4, i8 4, i8 5, i8 5, i8 6, i8 6, i8 7, i8 7>, [[BROADCAST_SPLAT4]]
; DEFAULT-NEXT: [[TMP4:%.*]] = add <16 x i8> [[TMP3]], [[TMP1]]
-; DEFAULT-NEXT: [[TMP5:%.*]] = lshr <16 x i8> [[VEC_IND1]], splat (i8 2)
-; DEFAULT-NEXT: [[TMP6:%.*]] = mul <16 x i8> [[TMP5]], [[BROADCAST_SPLAT6]]
+; DEFAULT-NEXT: [[TMP6:%.*]] = mul <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 2, i8 2, i8 2, i8 2, i8 3, i8 3, i8 3, i8 3>, [[BROADCAST_SPLAT6]]
; DEFAULT-NEXT: [[TMP7:%.*]] = add <16 x i8> [[TMP4]], [[TMP6]]
-; DEFAULT-NEXT: [[TMP8:%.*]] = extractelement <16 x i1> [[TMP0]], i32 0
-; DEFAULT-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; DEFAULT: [[PRED_STORE_IF]]:
-; DEFAULT-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0
-; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP9]]
+; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0
; DEFAULT-NEXT: [[TMP11:%.*]] = extractelement <16 x i8> [[TMP7]], i32 0
; DEFAULT-NEXT: store i8 [[TMP11]], ptr [[TMP10]], align 1
; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]]
; DEFAULT: [[PRED_STORE_CONTINUE]]:
-; DEFAULT-NEXT: [[TMP12:%.*]] = extractelement <16 x i1> [[TMP0]], i32 1
-; DEFAULT-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
-; DEFAULT: [[PRED_STORE_IF6]]:
-; DEFAULT-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 1
-; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP13]]
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; DEFAULT: [[PRED_STORE_IF5]]:
+; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
; DEFAULT-NEXT: [[TMP15:%.*]] = extractelement <16 x i8> [[TMP7]], i32 1
; DEFAULT-NEXT: store i8 [[TMP15]], ptr [[TMP14]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE7]]
-; DEFAULT: [[PRED_STORE_CONTINUE7]]:
-; DEFAULT-NEXT: [[TMP16:%.*]] = extractelement <16 x i1> [[TMP0]], i32 2
-; DEFAULT-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
-; DEFAULT: [[PRED_STORE_IF8]]:
-; DEFAULT-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], 2
-; DEFAULT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP17]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; DEFAULT: [[PRED_STORE_CONTINUE6]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; DEFAULT: [[PRED_STORE_IF7]]:
+; DEFAULT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
; DEFAULT-NEXT: [[TMP19:%.*]] = extractelement <16 x i8> [[TMP7]], i32 2
; DEFAULT-NEXT: store i8 [[TMP19]], ptr [[TMP18]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE9]]
-; DEFAULT: [[PRED_STORE_CONTINUE9]]:
-; DEFAULT-NEXT: [[TMP20:%.*]] = extractelement <16 x i1> [[TMP0]], i32 3
-; DEFAULT-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
-; DEFAULT: [[PRED_STORE_IF10]]:
-; DEFAULT-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 3
-; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP21]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; DEFAULT: [[PRED_STORE_CONTINUE8]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; DEFAULT: [[PRED_STORE_IF9]]:
+; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
; DEFAULT-NEXT: [[TMP23:%.*]] = extractelement <16 x i8> [[TMP7]], i32 3
; DEFAULT-NEXT: store i8 [[TMP23]], ptr [[TMP22]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE11]]
-; DEFAULT: [[PRED_STORE_CONTINUE11]]:
-; DEFAULT-NEXT: [[TMP24:%.*]] = extractelement <16 x i1> [[TMP0]], i32 4
-; DEFAULT-NEXT: br i1 [[TMP24]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
-; DEFAULT: [[PRED_STORE_IF12]]:
-; DEFAULT-NEXT: [[TMP25:%.*]] = add i64 [[INDEX]], 4
-; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP25]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; DEFAULT: [[PRED_STORE_CONTINUE10]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; DEFAULT: [[PRED_STORE_IF11]]:
+; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
; DEFAULT-NEXT: [[TMP27:%.*]] = extractelement <16 x i8> [[TMP7]], i32 4
; DEFAULT-NEXT: store i8 [[TMP27]], ptr [[TMP26]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE13]]
-; DEFAULT: [[PRED_STORE_CONTINUE13]]:
-; DEFAULT-NEXT: [[TMP28:%.*]] = extractelement <16 x i1> [[TMP0]], i32 5
-; DEFAULT-NEXT: br i1 [[TMP28]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]]
-; DEFAULT: [[PRED_STORE_IF14]]:
-; DEFAULT-NEXT: [[TMP29:%.*]] = add i64 [[INDEX]], 5
-; DEFAULT-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP29]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; DEFAULT: [[PRED_STORE_CONTINUE12]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; DEFAULT: [[PRED_STORE_IF13]]:
+; DEFAULT-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
; DEFAULT-NEXT: [[TMP31:%.*]] = extractelement <16 x i8> [[TMP7]], i32 5
; DEFAULT-NEXT: store i8 [[TMP31]], ptr [[TMP30]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE15]]
-; DEFAULT: [[PRED_STORE_CONTINUE15]]:
-; DEFAULT-NEXT: [[TMP32:%.*]] = extractelement <16 x i1> [[TMP0]], i32 6
-; DEFAULT-NEXT: br i1 [[TMP32]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17:.*]]
-; DEFAULT: [[PRED_STORE_IF16]]:
-; DEFAULT-NEXT: [[TMP33:%.*]] = add i64 [[INDEX]], 6
-; DEFAULT-NEXT: [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP33]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE14]]
+; DEFAULT: [[PRED_STORE_CONTINUE14]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
+; DEFAULT: [[PRED_STORE_IF15]]:
+; DEFAULT-NEXT: [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
; DEFAULT-NEXT: [[TMP35:%.*]] = extractelement <16 x i8> [[TMP7]], i32 6
; DEFAULT-NEXT: store i8 [[TMP35]], ptr [[TMP34]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE17]]
-; DEFAULT: [[PRED_STORE_CONTINUE17]]:
-; DEFAULT-NEXT: [[TMP36:%.*]] = extractelement <16 x i1> [[TMP0]], i32 7
-; DEFAULT-NEXT: br i1 [[TMP36]], label %[[PRED_STORE_IF18:.*]], label %[[PRED_STORE_CONTINUE19:.*]]
-; DEFAULT: [[PRED_STORE_IF18]]:
-; DEFAULT-NEXT: [[TMP37:%.*]] = add i64 [[INDEX]], 7
-; DEFAULT-NEXT: [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP37]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; DEFAULT: [[PRED_STORE_CONTINUE16]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
+; DEFAULT: [[PRED_STORE_IF17]]:
+; DEFAULT-NEXT: [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
; DEFAULT-NEXT: [[TMP39:%.*]] = extractelement <16 x i8> [[TMP7]], i32 7
; DEFAULT-NEXT: store i8 [[TMP39]], ptr [[TMP38]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE19]]
-; DEFAULT: [[PRED_STORE_CONTINUE19]]:
-; DEFAULT-NEXT: [[TMP40:%.*]] = extractelement <16 x i1> [[TMP0]], i32 8
-; DEFAULT-NEXT: br i1 [[TMP40]], label %[[PRED_STORE_IF20:.*]], label %[[PRED_STORE_CONTINUE21:.*]]
-; DEFAULT: [[PRED_STORE_IF20]]:
-; DEFAULT-NEXT: [[TMP41:%.*]] = add i64 [[INDEX]], 8
-; DEFAULT-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP41]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE18]]
+; DEFAULT: [[PRED_STORE_CONTINUE18]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
+; DEFAULT: [[PRED_STORE_IF19]]:
+; DEFAULT-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
; DEFAULT-NEXT: [[TMP43:%.*]] = extractelement <16 x i8> [[TMP7]], i32 8
; DEFAULT-NEXT: store i8 [[TMP43]], ptr [[TMP42]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE21]]
-; DEFAULT: [[PRED_STORE_CONTINUE21]]:
-; DEFAULT-NEXT: [[TMP44:%.*]] = extractelement <16 x i1> [[TMP0]], i32 9
-; DEFAULT-NEXT: br i1 [[TMP44]], label %[[PRED_STORE_IF22:.*]], label %[[PRED_STORE_CONTINUE23:.*]]
-; DEFAULT: [[PRED_STORE_IF22]]:
-; DEFAULT-NEXT: [[TMP45:%.*]] = add i64 [[INDEX]], 9
-; DEFAULT-NEXT: [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP45]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE20]]
+; DEFAULT: [[PRED_STORE_CONTINUE20]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
+; DEFAULT: [[PRED_STORE_IF21]]:
+; DEFAULT-NEXT: [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 9
; DEFAULT-NEXT: [[TMP47:%.*]] = extractelement <16 x i8> [[TMP7]], i32 9
; DEFAULT-NEXT: store i8 [[TMP47]], ptr [[TMP46]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE23]]
-; DEFAULT: [[PRED_STORE_CONTINUE23]]:
-; DEFAULT-NEXT: [[TMP48:%.*]] = extractelement <16 x i1> [[TMP0]], i32 10
-; DEFAULT-NEXT: br i1 [[TMP48]], label %[[PRED_STORE_IF24:.*]], label %[[PRED_STORE_CONTINUE25:.*]]
-; DEFAULT: [[PRED_STORE_IF24]]:
-; DEFAULT-NEXT: [[TMP49:%.*]] = add i64 [[INDEX]], 10
-; DEFAULT-NEXT: [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP49]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE22]]
+; DEFAULT: [[PRED_STORE_CONTINUE22]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
+; DEFAULT: [[PRED_STORE_IF23]]:
+; DEFAULT-NEXT: [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 10
; DEFAULT-NEXT: [[TMP51:%.*]] = extractelement <16 x i8> [[TMP7]], i32 10
; DEFAULT-NEXT: store i8 [[TMP51]], ptr [[TMP50]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE25]]
-; DEFAULT: [[PRED_STORE_CONTINUE25]]:
-; DEFAULT-NEXT: [[TMP52:%.*]] = extractelement <16 x i1> [[TMP0]], i32 11
-; DEFAULT-NEXT: br i1 [[TMP52]], label %[[PRED_STORE_IF26:.*]], label %[[PRED_STORE_CONTINUE27:.*]]
-; DEFAULT: [[PRED_STORE_IF26]]:
-; DEFAULT-NEXT: [[TMP53:%.*]] = add i64 [[INDEX]], 11
-; DEFAULT-NEXT: [[TMP54:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP53]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE24]]
+; DEFAULT: [[PRED_STORE_CONTINUE24]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
+; DEFAULT: [[PRED_STORE_IF25]]:
+; DEFAULT-NEXT: [[TMP54:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 11
; DEFAULT-NEXT: [[TMP55:%.*]] = extractelement <16 x i8> [[TMP7]], i32 11
; DEFAULT-NEXT: store i8 [[TMP55]], ptr [[TMP54]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE27]]
-; DEFAULT: [[PRED_STORE_CONTINUE27]]:
-; DEFAULT-NEXT: [[TMP56:%.*]] = extractelement <16 x i1> [[TMP0]], i32 12
-; DEFAULT-NEXT: br i1 [[TMP56]], label %[[PRED_STORE_IF28:.*]], label %[[PRED_STORE_CONTINUE29:.*]]
-; DEFAULT: [[PRED_STORE_IF28]]:
-; DEFAULT-NEXT: [[TMP57:%.*]] = add i64 [[INDEX]], 12
-; DEFAULT-NEXT: [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP57]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE26]]
+; DEFAULT: [[PRED_STORE_CONTINUE26]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
+; DEFAULT: [[PRED_STORE_IF27]]:
+; DEFAULT-NEXT: [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
; DEFAULT-NEXT: [[TMP59:%.*]] = extractelement <16 x i8> [[TMP7]], i32 12
; DEFAULT-NEXT: store i8 [[TMP59]], ptr [[TMP58]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE29]]
-; DEFAULT: [[PRED_STORE_CONTINUE29]]:
-; DEFAULT-NEXT: [[TMP60:%.*]] = extractelement <16 x i1> [[TMP0]], i32 13
-; DEFAULT-NEXT: br i1 [[TMP60]], label %[[PRED_STORE_IF30:.*]], label %[[PRED_STORE_CONTINUE31:.*]]
-; DEFAULT: [[PRED_STORE_IF30]]:
-; DEFAULT-NEXT: [[TMP61:%.*]] = add i64 [[INDEX]], 13
-; DEFAULT-NEXT: [[TMP62:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP61]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE28]]
+; DEFAULT: [[PRED_STORE_CONTINUE28]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
+; DEFAULT: [[PRED_STORE_IF29]]:
+; DEFAULT-NEXT: [[TMP62:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 13
; DEFAULT-NEXT: [[TMP63:%.*]] = extractelement <16 x i8> [[TMP7]], i32 13
; DEFAULT-NEXT: store i8 [[TMP63]], ptr [[TMP62]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE31]]
-; DEFAULT: [[PRED_STORE_CONTINUE31]]:
-; DEFAULT-NEXT: [[TMP64:%.*]] = extractelement <16 x i1> [[TMP0]], i32 14
-; DEFAULT-NEXT: br i1 [[TMP64]], label %[[PRED_STORE_IF32:.*]], label %[[PRED_STORE_CONTINUE33:.*]]
-; DEFAULT: [[PRED_STORE_IF32]]:
-; DEFAULT-NEXT: [[TMP65:%.*]] = add i64 [[INDEX]], 14
-; DEFAULT-NEXT: [[TMP66:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP65]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE30]]
+; DEFAULT: [[PRED_STORE_CONTINUE30]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
+; DEFAULT: [[PRED_STORE_IF31]]:
+; DEFAULT-NEXT: [[TMP66:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 14
; DEFAULT-NEXT: [[TMP67:%.*]] = extractelement <16 x i8> [[TMP7]], i32 14
; DEFAULT-NEXT: store i8 [[TMP67]], ptr [[TMP66]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE33]]
-; DEFAULT: [[PRED_STORE_CONTINUE33]]:
-; DEFAULT-NEXT: [[TMP68:%.*]] = extractelement <16 x i1> [[TMP0]], i32 15
-; DEFAULT-NEXT: br i1 [[TMP68]], label %[[PRED_STORE_IF34:.*]], label %[[PRED_STORE_CONTINUE35]]
-; DEFAULT: [[PRED_STORE_IF34]]:
-; DEFAULT-NEXT: [[TMP69:%.*]] = add i64 [[INDEX]], 15
-; DEFAULT-NEXT: [[TMP70:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP69]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; DEFAULT: [[PRED_STORE_CONTINUE32]]:
+; DEFAULT-NEXT: br i1 false, label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
+; DEFAULT: [[PRED_STORE_IF33]]:
+; DEFAULT-NEXT: [[TMP70:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 15
; DEFAULT-NEXT: [[TMP71:%.*]] = extractelement <16 x i8> [[TMP7]], i32 15
; DEFAULT-NEXT: store i8 [[TMP71]], ptr [[TMP70]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE35]]
-; DEFAULT: [[PRED_STORE_CONTINUE35]]:
-; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <16 x i8> [[VEC_IND]], splat (i8 16)
-; DEFAULT-NEXT: [[VEC_IND_NEXT36]] = add <16 x i8> [[VEC_IND1]], splat (i8 16)
-; DEFAULT-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE34]]
+; DEFAULT: [[PRED_STORE_CONTINUE34]]:
+; DEFAULT-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; DEFAULT: [[FOR_COND_CLEANUP]]:
@@ -471,7 +430,7 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
; DEFAULT-NEXT: [[TMP24:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
; DEFAULT-NEXT: [[TMP23:%.*]] = xor i1 [[TMP24]], true
; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[DOTSPLAT]]
-; DEFAULT-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; DEFAULT-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; DEFAULT: [[FOR_COND_CLEANUP]]:
@@ -621,7 +580,7 @@ define void @dont_vectorize_with_minsize() {
; DEFAULT-NEXT: store <8 x i16> [[TMP15]], ptr [[TMP11]], align 2
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; DEFAULT-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; DEFAULT-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; DEFAULT-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; DEFAULT: [[FOR_COND_CLEANUP]]:
@@ -737,7 +696,7 @@ define void @vectorization_forced_minsize_reduce_width() {
; DEFAULT-NEXT: store <8 x i16> [[TMP15]], ptr [[TMP11]], align 2
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; DEFAULT-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; DEFAULT-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; DEFAULT-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; DEFAULT: [[FOR_COND_CLEANUP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
index 2c82cae589036..e93a5271acf24 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
@@ -587,25 +587,19 @@ define double @test_scalarization_cost_for_load_of_address(ptr %src.0, ptr %src.
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi double [ 3.000000e+00, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr [[T:%.*]], ptr [[SRC_0]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <6 x double>, ptr [[GEP_0]], align 8
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <6 x double>, ptr [[SRC_0]], align 8
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 1, i32 4>
; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[STRIDED_VEC]], splat (double 3.000000e+00)
-; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x double> [[STRIDED_VEC1]], splat (double 3.000000e+00)
-; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[STRIDED_VEC2]], splat (double 3.000000e+00)
-; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[GEP_SRC]], align 8
-; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x double> [[TMP7]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr [[T_2:%.*]], ptr [[SRC_2]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr [[T_2]], ptr [[SRC_2]], i64 [[TMP1]]
-; CHECK-NEXT: [[GEP_72:%.*]] = getelementptr i8, ptr [[GEP_SRC_2]], i64 72
+; CHECK-NEXT: [[TMP0:%.*]] = fmul <2 x double> [[STRIDED_VEC]], splat (double 3.000000e+00)
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x double> [[STRIDED_VEC1]], splat (double 3.000000e+00)
+; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x double> [[STRIDED_VEC2]], splat (double 3.000000e+00)
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[SRC_1]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x double> [[TMP4]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr [[T_2:%.*]], ptr [[SRC_2]], i64 1
+; CHECK-NEXT: [[GEP_72:%.*]] = getelementptr i8, ptr [[SRC_2]], i64 72
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP11]], i64 72
; CHECK-NEXT: [[L_P_2:%.*]] = load ptr, ptr [[GEP_72]], align 8
; CHECK-NEXT: [[TMP15:%.*]] = load ptr, ptr [[TMP13]], align 8
@@ -614,9 +608,8 @@ define double @test_scalarization_cost_for_load_of_address(ptr %src.0, ptr %src.
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x double> poison, double [[LV]], i32 0
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x double> [[TMP18]], double [[TMP17]], i32 1
; CHECK-NEXT: [[TMP20:%.*]] = fmul <2 x double> [[TMP9]], [[TMP19]]
-; CHECK-NEXT: [[TMP21]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> [[TMP20]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: [[TMP21:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double 3.000000e+00, <2 x double> [[TMP20]])
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -776,7 +769,7 @@ define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {
; CHECK-NEXT: [[TMP66]] = or <16 x i32> [[TMP65]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP66]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
index 0f18b29b25d6d..1a0c3afe500e8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
@@ -552,11 +552,7 @@ define double @test_scalarization_cost_for_load_of_address(ptr %src.0, ptr %src.
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi double [ 3.000000e+00, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr [[T:%.*]], ptr [[SRC_0]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <6 x double>, ptr [[GEP_0]], align 8
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <6 x double>, ptr [[SRC_0]], align 8
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 1, i32 4>
; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 2, i32 5>
@@ -565,23 +561,20 @@ define double @test_scalarization_cost_for_load_of_address(ptr %src.0, ptr %src.
; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[STRIDED_VEC2]], splat (double 3.000000e+00)
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[SRC_1]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x double> [[TMP7]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr [[T_2:%.*]], ptr [[SRC_2]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr [[T_2]], ptr [[SRC_2]], i64 [[TMP1]]
-; CHECK-NEXT: [[GEP_72:%.*]] = getelementptr i8, ptr [[GEP_SRC_2]], i64 72
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP11]], i64 72
-; CHECK-NEXT: [[L_P_2:%.*]] = load ptr, ptr [[GEP_72]], align 8
-; CHECK-NEXT: [[TMP15:%.*]] = load ptr, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[LV:%.*]] = load double, ptr [[L_P_2]], align 8
-; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP15]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr [[T_2:%.*]], ptr [[SRC_2]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC_2]], i64 72
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP11]], i64 72
+; CHECK-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP8]], align 8
+; CHECK-NEXT: [[LV:%.*]] = load double, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP10]], align 8
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x double> poison, double [[LV]], i32 0
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x double> [[TMP18]], double [[TMP17]], i32 1
; CHECK-NEXT: [[TMP20:%.*]] = fmul <2 x double> [[TMP9]], [[TMP19]]
-; CHECK-NEXT: [[TMP21]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> [[TMP20]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: [[TMP21:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double 3.000000e+00, <2 x double> [[TMP20]])
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -741,7 +734,7 @@ define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {
; CHECK-NEXT: [[TMP66]] = or <16 x i32> [[TMP65]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP66]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
index 78b54efe7e682..0af9795cfdf62 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
@@ -285,13 +285,9 @@ define i8 @mul_non_pow_2_low_trip_count(ptr noalias %a) {
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i8> [ <i8 2, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; CHECK-NEXT: [[TMP1]] = mul <8 x i8> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0:%.*]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = mul <8 x i8> [[WIDE_LOAD]], <i8 2, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
; CHECK: middle.block:
; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> [[TMP1]])
; CHECK-NEXT: br label [[SCALAR_PH:%.*]]
@@ -300,12 +296,12 @@ define i8 @mul_non_pow_2_low_trip_count(ptr noalias %a) {
; CHECK: for.body:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 8, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[RDX:%.*]] = phi i8 [ [[TMP3]], [[SCALAR_PH]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[TMP0]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP]], align 1
; CHECK-NEXT: [[MUL]] = mul i8 [[TMP5]], [[RDX]]
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 10
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: for.end:
; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i8 [ [[MUL]], [[FOR_BODY]] ]
; CHECK-NEXT: ret i8 [[MUL_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vector-loop-backedge-elimination-with-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vector-loop-backedge-elimination-with-evl.ll
index f1dda3d5a2f91..a80d5dacff583 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vector-loop-backedge-elimination-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vector-loop-backedge-elimination-with-evl.ll
@@ -41,13 +41,9 @@ define i32 @test_remove_iv(i32 %start) #0 {
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 6, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[TMP4:%.*]] = xor <vscale x 4 x i32> [[VEC_PHI]], splat (i32 3)
-; CHECK-NEXT: [[TMP5]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP3]])
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]]
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor <vscale x 4 x i32> [[TMP2]], splat (i32 3)
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP2]], i32 6)
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP5]])
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -71,8 +67,3 @@ exit:
attributes #0 = { vscale_range(2,2) }
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 58b21f9bc816d..e0939fe5244eb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -603,19 +603,19 @@ define i64 @live_in_known_1_via_scev() {
; CHECK-NEXT: [[P:%.*]] = phi i32 [ 1, %[[ENTRY]] ]
; CHECK-NEXT: [[N:%.*]] = add nuw nsw i32 [[SEL]], 6
; CHECK-NEXT: [[P_EXT:%.*]] = zext nneg i32 [[P]] to i64
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ <i64 3, i64 1, i64 1, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_PHI]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ <i64 3, i64 1, i64 1, i64 1>, %[[VECTOR_BODY]] ], [ [[VEC_PHI]], %[[EXIT]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> [[VEC_PHI]])
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret i64 [[TMP3]]
;
entry:
@@ -657,9 +657,8 @@ define i64 @cost_loop_invariant_recipes(i1 %x, i64 %y) {
; CHECK-NEXT: [[TMP2:%.*]] = shl <2 x i64> [[BROADCAST_SPLAT]], [[TMP1]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ splat (i64 1), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3]] = mul <2 x i64> [[TMP2]], [[VEC_PHI]]
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul <2 x i64> [[TMP2]], splat (i64 1)
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> [[TMP3]])
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -697,18 +696,15 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = and <16 x i32> [[VEC_PHI]], splat (i32 1)
-; CHECK-NEXT: [[TMP1:%.*]] = or <16 x i32> [[TMP0]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP2:%.*]] = trunc <16 x i32> [[TMP1]] to <16 x i1>
-; CHECK-NEXT: [[TMP3]] = zext <16 x i1> [[TMP2]] to <16 x i32>
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT: [[TMP0:%.*]] = or <16 x i32> zeroinitializer, [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc <16 x i32> [[TMP0]] to <16 x i1>
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP2]])
-; CHECK-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i32 [[TMP5]]
+; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
%conv = zext i1 %cmp to i32
@@ -778,7 +774,7 @@ define i32 @g(i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i32> [[TMP16]], [[TMP15]]
; CHECK-NEXT: [[BIN_RDX5:%.*]] = or <4 x i32> [[TMP17]], [[BIN_RDX]]
@@ -812,7 +808,7 @@ define i32 @g(i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT15]] = add nuw i32 [[INDEX9]], 4
; CHECK-NEXT: [[VEC_IND_NEXT11]] = add <4 x i32> [[VEC_IND10]], splat (i32 4)
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT15]], [[N_VEC8]]
-; CHECK-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP25]])
; CHECK-NEXT: [[CMP_N18:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC8]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-ordered-reduction.ll b/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-ordered-reduction.ll
index 2e946693af294..b0ff3989666c4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-ordered-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-ordered-reduction.ll
@@ -273,22 +273,18 @@ define float @ordered_reduction_epilogue_dead_main_loop(ptr %p, i64 %n) "prefer-
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[BOUND]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 16
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[TMP0]], i64 32
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[TMP0]], i64 48
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[P]], i64 16
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[P]], i64 32
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[P]], i64 48
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[P]], align 4
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[VEC_PHI]], <16 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP4]], <16 x float> [[WIDE_LOAD1]])
; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP5]], <16 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[TMP7]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP6]], <16 x float> [[WIDE_LOAD3]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP6]], <16 x float> [[WIDE_LOAD3]])
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[BOUND]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -296,35 +292,32 @@ define float @ordered_reduction_epilogue_dead_main_loop(ptr %p, i64 %n) "prefer-
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[BOUND]], 16
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[BOUND]], [[N_MOD_VF4]]
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[VEC_EPILOG_PH]] ], [ [[TMP9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[P]], i64 [[INDEX6]]
; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x float>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP9]] = call float @llvm.vector.reduce.fadd.v16f32(float [[VEC_PHI7]], <16 x float> [[WIDE_LOAD8]])
-; CHECK-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX6]], 16
-; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[VEC_PHI7]], <16 x float> [[WIDE_LOAD8]])
+; CHECK-NEXT: br label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[BOUND]], [[N_VEC5]]
; CHECK-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi float [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi float [ [[BC_MERGE_RDX11]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[FADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[FADD:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[LOAD:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT: [[FADD]] = fadd float [[RED]], [[LOAD]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[BOUND]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi float [ [[FADD]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[RES]]
@@ -364,22 +357,18 @@ define float @ordered_reduction_nonzero_start_dead_main_vector_loop(ptr %p, i64
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[BOUND]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 4.200000e+01, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 16
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[TMP0]], i64 32
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[TMP0]], i64 48
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[P]], i64 16
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[P]], i64 32
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[P]], i64 48
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[P]], align 4
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[VEC_PHI]], <16 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float 4.200000e+01, <16 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP4]], <16 x float> [[WIDE_LOAD1]])
; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP5]], <16 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[TMP7]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP6]], <16 x float> [[WIDE_LOAD3]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP6]], <16 x float> [[WIDE_LOAD3]])
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[BOUND]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -387,35 +376,32 @@ define float @ordered_reduction_nonzero_start_dead_main_vector_loop(ptr %p, i64
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 4.200000e+01, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 4.200000e+01, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[BOUND]], 16
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[BOUND]], [[N_MOD_VF4]]
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[VEC_EPILOG_PH]] ], [ [[TMP9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[P]], i64 [[INDEX6]]
; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x float>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP9]] = call float @llvm.vector.reduce.fadd.v16f32(float [[VEC_PHI7]], <16 x float> [[WIDE_LOAD8]])
-; CHECK-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX6]], 16
-; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[VEC_PHI7]], <16 x float> [[WIDE_LOAD8]])
+; CHECK-NEXT: br label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[BOUND]], [[N_VEC5]]
; CHECK-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi float [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 4.200000e+01, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 4.200000e+01, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi float [ [[BC_MERGE_RDX11]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[FADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[FADD:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[LOAD:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT: [[FADD]] = fadd float [[RED]], [[LOAD]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[BOUND]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi float [ [[FADD]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[RES]]
@@ -454,35 +440,29 @@ define { float, float } @two_ordered_reductions(ptr %p, ptr %q, i64 %n) "prefer-
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[BOUND]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi float [ 1.000000e+00, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 16
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[TMP0]], i64 32
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[TMP0]], i64 48
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[P]], i64 16
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[P]], i64 32
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr float, ptr [[P]], i64 48
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[P]], align 4
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr float, ptr [[Q]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[TMP4]], i64 16
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[TMP4]], i64 32
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr float, ptr [[TMP4]], i64 48
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[Q]], i64 16
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[Q]], i64 32
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr float, ptr [[Q]], i64 48
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x float>, ptr [[Q]], align 4
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x float>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x float>, ptr [[TMP10]], align 4
; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x float>, ptr [[TMP20]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP6]], <16 x float> [[WIDE_LOAD1]])
-; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP7]], <16 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP8]], <16 x float> [[WIDE_LOAD3]])
-; CHECK-NEXT: [[TMP18]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP9]], <16 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP12:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP11]], <16 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP9]], <16 x float> [[WIDE_LOAD5]])
+; CHECK-NEXT: [[TMP11:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP18]], <16 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP12:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float 1.000000e+00, <16 x float> [[WIDE_LOAD4]])
; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP12]], <16 x float> [[WIDE_LOAD7]])
; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP13]], <16 x float> [[WIDE_LOAD9]])
-; CHECK-NEXT: [[TMP19]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP21]], <16 x float> [[WIDE_LOAD8]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[TMP21]], <16 x float> [[WIDE_LOAD8]])
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[BOUND]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -490,36 +470,32 @@ define { float, float } @two_ordered_reductions(ptr %p, ptr %q, i64 %n) "prefer-
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX1:%.*]] = phi float [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX9:%.*]] = phi float [ [[TMP19]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP19]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF9:%.*]] = urem i64 [[BOUND]], 16
; CHECK-NEXT: [[N_VEC10:%.*]] = sub i64 [[BOUND]], [[N_MOD_VF9]]
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL1]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[BC_MERGE_RDX1]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[BC_MERGE_RDX9]], %[[VEC_EPILOG_PH]] ], [ [[TMP17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr float, ptr [[P]], i64 [[VEC_EPILOG_RESUME_VAL]]
; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x float>, ptr [[TMP14]], align 4
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr float, ptr [[Q]], i64 [[VEC_EPILOG_RESUME_VAL]]
; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x float>, ptr [[TMP15]], align 4
-; CHECK-NEXT: [[TMP16]] = call float @llvm.vector.reduce.fadd.v16f32(float [[BC_MERGE_RDX]], <16 x float> [[WIDE_LOAD11]])
-; CHECK-NEXT: [[TMP17]] = call float @llvm.vector.reduce.fadd.v16f32(float [[BC_MERGE_RDX8]], <16 x float> [[WIDE_LOAD12]])
-; CHECK-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[VEC_EPILOG_RESUME_VAL]], 16
-; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT: [[TMP16:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[BC_MERGE_RDX]], <16 x float> [[WIDE_LOAD11]])
+; CHECK-NEXT: [[TMP17:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[BC_MERGE_RDX8]], <16 x float> [[WIDE_LOAD12]])
+; CHECK-NEXT: br label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[BOUND]], [[N_VEC10]]
; CHECK-NEXT: br i1 [[CMP_N13]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX19:%.*]] = phi float [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX20:%.*]] = phi float [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX14:%.*]] = phi float [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX15:%.*]] = phi float [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RED1:%.*]] = phi float [ [[BC_MERGE_RDX19]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[FADD1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RED2:%.*]] = phi float [ [[BC_MERGE_RDX20]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[FADD2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED1:%.*]] = phi float [ [[BC_MERGE_RDX14]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[FADD1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED2:%.*]] = phi float [ [[BC_MERGE_RDX15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[FADD2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[FADD1]] = fadd float [[RED1]], [[LOAD1]]
@@ -528,9 +504,9 @@ define { float, float } @two_ordered_reductions(ptr %p, ptr %q, i64 %n) "prefer-
; CHECK-NEXT: [[FADD2]] = fadd float [[RED2]], [[LOAD2]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[BOUND]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R1:%.*]] = phi float [ [[FADD1]], %[[LOOP]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ], [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[R1:%.*]] = phi float [ [[FADD1]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[R2:%.*]] = phi float [ [[FADD2]], %[[LOOP]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[RET:%.*]] = insertvalue { float, float } undef, float [[R1]], 0
; CHECK-NEXT: [[RET2:%.*]] = insertvalue { float, float } [[RET]], float [[R2]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
index 8f347ebf87016..2cfb521e71b18 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
@@ -2455,44 +2455,22 @@ define i32 @test_non_unit_stride_five(i64 %len, ptr %test_base) {
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP112:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP113:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP114:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP115:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 5
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 5
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 10
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 15
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 20
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 25
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 35
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 40
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 45
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 50
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 55
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 60
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 65
-; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 70
-; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 75
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP14]]
-; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[TMP15]]
-; CHECK-NEXT: [[TMP32:%.*]] = load i1, ptr [[TMP16]], align 1
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE:%.*]], i64 5
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 10
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 15
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 20
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 25
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 30
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 35
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 40
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 45
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 50
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 55
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 60
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 65
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 70
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 75
+; CHECK-NEXT: [[TMP32:%.*]] = load i1, ptr [[TEST_BASE]], align 1
; CHECK-NEXT: [[TMP33:%.*]] = load i1, ptr [[TMP17]], align 1
; CHECK-NEXT: [[TMP34:%.*]] = load i1, ptr [[TMP18]], align 1
; CHECK-NEXT: [[TMP35:%.*]] = load i1, ptr [[TMP19]], align 1
@@ -2524,30 +2502,29 @@ define i32 @test_non_unit_stride_five(i64 %len, ptr %test_base) {
; CHECK-NEXT: [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
; CHECK-NEXT: [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
; CHECK-NEXT: [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
-; CHECK-NEXT: [[TMP64:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP66:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP68:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP70:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP71:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP72:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP73:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP76:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP77:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP14]]
-; CHECK-NEXT: [[TMP79:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP15]]
-; CHECK-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP64]], align 4
-; CHECK-NEXT: [[TMP81:%.*]] = load i32, ptr [[TMP65]], align 4
-; CHECK-NEXT: [[TMP82:%.*]] = load i32, ptr [[TMP66]], align 4
+; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 5
+; CHECK-NEXT: [[TMP64:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 10
+; CHECK-NEXT: [[TMP66:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 15
+; CHECK-NEXT: [[TMP68:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 20
+; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 25
+; CHECK-NEXT: [[TMP70:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 30
+; CHECK-NEXT: [[TMP71:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 35
+; CHECK-NEXT: [[TMP72:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 40
+; CHECK-NEXT: [[TMP73:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 45
+; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 50
+; CHECK-NEXT: [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 55
+; CHECK-NEXT: [[TMP76:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 60
+; CHECK-NEXT: [[TMP77:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 65
+; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 70
+; CHECK-NEXT: [[TMP79:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 75
+; CHECK-NEXT: [[TMP80:%.*]] = load i32, ptr [[ALLOCA]], align 4
; CHECK-NEXT: [[TMP83:%.*]] = load i32, ptr [[TMP67]], align 4
+; CHECK-NEXT: [[TMP82:%.*]] = load i32, ptr [[TMP64]], align 4
+; CHECK-NEXT: [[TMP65:%.*]] = load i32, ptr [[TMP66]], align 4
; CHECK-NEXT: [[TMP84:%.*]] = insertelement <4 x i32> poison, i32 [[TMP80]], i32 0
-; CHECK-NEXT: [[TMP85:%.*]] = insertelement <4 x i32> [[TMP84]], i32 [[TMP81]], i32 1
+; CHECK-NEXT: [[TMP85:%.*]] = insertelement <4 x i32> [[TMP84]], i32 [[TMP83]], i32 1
; CHECK-NEXT: [[TMP86:%.*]] = insertelement <4 x i32> [[TMP85]], i32 [[TMP82]], i32 2
-; CHECK-NEXT: [[TMP87:%.*]] = insertelement <4 x i32> [[TMP86]], i32 [[TMP83]], i32 3
+; CHECK-NEXT: [[TMP87:%.*]] = insertelement <4 x i32> [[TMP86]], i32 [[TMP65]], i32 3
; CHECK-NEXT: [[TMP88:%.*]] = load i32, ptr [[TMP68]], align 4
; CHECK-NEXT: [[TMP89:%.*]] = load i32, ptr [[TMP69]], align 4
; CHECK-NEXT: [[TMP90:%.*]] = load i32, ptr [[TMP70]], align 4
@@ -2576,16 +2553,12 @@ define i32 @test_non_unit_stride_five(i64 %len, ptr %test_base) {
; CHECK-NEXT: [[PREDPHI4:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[TMP95]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[PREDPHI5:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[TMP103]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[PREDPHI6:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[TMP111]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP112]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
-; CHECK-NEXT: [[TMP113]] = add <4 x i32> [[VEC_PHI1]], [[PREDPHI4]]
-; CHECK-NEXT: [[TMP114]] = add <4 x i32> [[VEC_PHI2]], [[PREDPHI5]]
-; CHECK-NEXT: [[TMP115]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI6]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: [[TMP112:%.*]] = add <4 x i32> zeroinitializer, [[PREDPHI]]
+; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
; CHECK: middle.block:
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP113]], [[TMP112]]
-; CHECK-NEXT: [[BIN_RDX7:%.*]] = add <4 x i32> [[TMP114]], [[BIN_RDX]]
-; CHECK-NEXT: [[BIN_RDX8:%.*]] = add <4 x i32> [[TMP115]], [[BIN_RDX7]]
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PREDPHI4]], [[TMP112]]
+; CHECK-NEXT: [[BIN_RDX4:%.*]] = add <4 x i32> [[PREDPHI5]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX8:%.*]] = add <4 x i32> [[PREDPHI6]], [[BIN_RDX4]]
; CHECK-NEXT: [[TMP117:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX8]])
; CHECK-NEXT: br label [[SCALAR_PH:%.*]]
; CHECK: scalar.ph:
@@ -2605,7 +2578,7 @@ define i32 @test_non_unit_stride_five(i64 %len, ptr %test_base) {
; CHECK-NEXT: [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
; CHECK-NEXT: [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
; CHECK-NEXT: [[EXIT:%.*]] = icmp ugt i64 [[IV]], 100
-; CHECK-NEXT: br i1 [[EXIT]], label [[LOOP_EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXIT]], label [[LOOP_EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: loop_exit:
; CHECK-NEXT: [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ]
; CHECK-NEXT: ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -2771,7 +2744,7 @@ define i32 @test_non_unit_stride_off_by_four_bytes(i64 %len, ptr %test_base) {
; CHECK-NEXT: [[TMP115]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP116:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP116]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP116]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP113]], [[TMP112]]
; CHECK-NEXT: [[BIN_RDX7:%.*]] = add <4 x i32> [[TMP114]], [[BIN_RDX]]
@@ -2795,7 +2768,7 @@ define i32 @test_non_unit_stride_off_by_four_bytes(i64 %len, ptr %test_base) {
; CHECK-NEXT: [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
; CHECK-NEXT: [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
; CHECK-NEXT: [[EXIT:%.*]] = icmp ugt i64 [[IV]], 100
-; CHECK-NEXT: br i1 [[EXIT]], label [[LOOP_EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXIT]], label [[LOOP_EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: loop_exit:
; CHECK-NEXT: [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ]
; CHECK-NEXT: ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -2978,7 +2951,7 @@ define i32 @test_non_unit_stride_with_first_iteration_step_access(i64 %len, ptr
; CHECK-NEXT: [[TMP131]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP132:%.*]] = icmp eq i64 [[INDEX_NEXT]], 144
-; CHECK-NEXT: br i1 [[TMP132]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP132]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP129]], [[TMP128]]
; CHECK-NEXT: [[BIN_RDX7:%.*]] = add <4 x i32> [[TMP130]], [[BIN_RDX]]
@@ -3002,7 +2975,7 @@ define i32 @test_non_unit_stride_with_first_iteration_step_access(i64 %len, ptr
; CHECK-NEXT: [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
; CHECK-NEXT: [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
; CHECK-NEXT: [[EXIT:%.*]] = icmp ugt i64 [[IV]], 300
-; CHECK-NEXT: br i1 [[EXIT]], label [[LOOP_EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXIT]], label [[LOOP_EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK: loop_exit:
; CHECK-NEXT: [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ]
; CHECK-NEXT: ret i32 [[ACCUM_NEXT_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index 76f30decf81e7..4a3356889c36e 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -777,10 +777,9 @@ define i16 @test_no_op_or_reduction_single_vector_iteration(i64 %N) {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[CLAMPED]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[VEC_PHI]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> zeroinitializer)
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[CLAMPED]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
@@ -794,23 +793,22 @@ define i16 @test_no_op_or_reduction_single_vector_iteration(i64 %N) {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i16> [ [[TMP1]], %[[VEC_EPILOG_PH]] ], [ [[VEC_PHI4]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT: br label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[VEC_PHI4]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP1]])
; CHECK-NEXT: [[CMP_N5:%.*]] = icmp eq i64 [[CLAMPED]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[CMP_N5]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i16 [ [[TMP2]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP0]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX5:%.*]] = phi i16 [ [[TMP2]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP0]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX6]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX5]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[RED_NEXT]] = or i16 [[RED]], 0
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[CLAMPED]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[TMP2]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i16 [[RED_NEXT_LCSSA]]
@@ -830,3 +828,183 @@ loop:
exit:
ret i16 %red.next
}
+
+; Test or-reduction with an induction-derived operand and a small bounded trip
+; count. The main vector loop is dissolved (trip count <= VF), and the epilogue
+; vector loop's reduction body remains with a ReductionStartVector operand.
+define i16 @test_or_reduction_with_induction_single_vector_iteration(i64 %N) {
+; CHECK-LABEL: define i16 @test_or_reduction_with_induction_single_vector_iteration(
+; CHECK-SAME: i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[CLAMPED:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 4)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[CLAMPED]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[CLAMPED]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[CLAMPED]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[CLAMPED]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> <i16 0, i16 1, i16 2, i16 3>)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[CLAMPED]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP0]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[CLAMPED]], 4
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[CLAMPED]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[N_VEC3]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i32 [[BC_RESUME_VAL]] to i16
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP4]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i16> [[BROADCAST_SPLAT]], <i16 0, i16 1, i16 2, i16 3>
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i16> [[TMP3]], [[INDUCTION]]
+; CHECK-NEXT: br label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP5]])
+; CHECK-NEXT: [[CMP_N4:%.*]] = icmp eq i64 [[CLAMPED]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[CMP_N4]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi i32 [ [[TMP2]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i16 [ [[TMP6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL5]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SHIFT:%.*]] = phi i32 [ [[BC_RESUME_VAL6]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SHIFT_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX7]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TRUNC:%.*]] = trunc i32 [[SHIFT]] to i16
+; CHECK-NEXT: [[RED_NEXT]] = or i16 [[RED]], [[TRUNC]]
+; CHECK-NEXT: [[SHIFT_NEXT]] = add i32 [[SHIFT]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[CLAMPED]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[TMP6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i16 [[RED_NEXT_LCSSA]]
+;
+entry:
+ %clamped = call i64 @llvm.umin.i32(i64 %N, i64 4)
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %shift = phi i32 [ 0, %entry ], [ %shift.next, %loop ]
+ %red = phi i16 [ 0, %entry ], [ %red.next, %loop ]
+ %trunc = trunc i32 %shift to i16
+ %red.next = or i16 %red, %trunc
+ %shift.next = add i32 %shift, 1
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %clamped
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i16 %red.next
+}
+
+; Test AnyOf reduction in epilogue with a small trip count that results in
+; the epilogue vector loop being dissolved (single iteration).
+define i32 @anyof_reduction_in_dissolved_epilogue(i32 %val, i1 %c) {
+; CHECK-LABEL: define i32 @anyof_reduction_in_dissolved_epilogue(
+; CHECK-SAME: i32 [[VAL:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[START:%.*]] = sext i1 [[C]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = zext i1 [[C]] to i32
+; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[START]]
+; CHECK-NEXT: br label %[[ITER_CHECK:.*]]
+; CHECK: [[OUTER_HEADER_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL:%.*]], %[[LOOP:.*]] ], [ [[RDX_SELECT:%.*]], %[[MIDDLE_BLOCK:.*]] ], [ [[RDX_SELECT7:%.*]], %[[VEC_EPILOG_MIDDLE_BLOCK:.*]] ]
+; CHECK-NEXT: br label %[[ITER_CHECK]]
+; CHECK: [[ITER_CHECK]]:
+; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SEL_LCSSA]], %[[OUTER_HEADER_LOOPEXIT]] ]
+; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i32 [[VAL]], 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT: [[RDX_SELECT]] = select i1 [[TMP4]], i32 [[OUTER_IV]], i32 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[OUTER_HEADER_LOOPEXIT]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[IND_END:%.*]] = add i32 [[START]], [[N_VEC]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 0
+; CHECK-NEXT: [[N_MOD_VF1:%.*]] = urem i32 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC2:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF1]]
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[START]], [[N_VEC2]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i32 [[VAL]], 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> poison, i1 [[TMP7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i1> poison, i1 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT3]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i1> [[BROADCAST_SPLAT4]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT: br label %[[VEC_EPILOG_MIDDLE_BLOCK]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP10:%.*]] = freeze i1 [[TMP9]]
+; CHECK-NEXT: [[RDX_SELECT7]] = select i1 [[TMP10]], i32 [[OUTER_IV]], i32 0
+; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC2]]
+; CHECK-NEXT: br i1 [[CMP_N8]], label %[[OUTER_HEADER_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX9:%.*]] = phi i32 [ [[RDX_SELECT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX9]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL]], %[[LOOP]] ]
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[RDX]], i32 [[OUTER_IV]]
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[TC:%.*]] = zext i1 [[C]] to i32
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[TC]]
+; CHECK-NEXT: br i1 [[EC]], label %[[OUTER_HEADER_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
+;
+entry:
+ %start = sext i1 %c to i32
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i32 [ 0, %entry ], [ %sel, %loop ]
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %start, %outer.header ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ 0, %outer.header ], [ %sel, %loop ]
+ %cmp = icmp eq i32 %val, 0
+ %sel = select i1 %cmp, i32 %rdx, i32 %outer.iv
+ %iv.next = add i32 %iv, 1
+ %tc = zext i1 %c to i32
+ %ec = icmp eq i32 %iv.next, %tc
+ br i1 %ec, label %outer.header, label %loop
+
+ uselistorder i32 %sel, { 1, 0 }
+}
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index dd3ad4d01b465..b1027a8540c57 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -194,182 +194,129 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
; IC4VF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
; IC4VF4-NEXT: br label %[[VECTOR_BODY:.*]]
; IC4VF4: [[VECTOR_BODY]]:
-; IC4VF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44:.*]] ]
-; IC4VF4-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP108:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP109:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP110:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP111:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
-; IC4VF4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 -4)
-; IC4VF4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i16> [[STEP_ADD_2]], splat (i16 -4)
-; IC4VF4-NEXT: [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
-; IC4VF4-NEXT: [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
-; IC4VF4-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
-; IC4VF4-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
-; IC4VF4-NEXT: [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 0, i32 1, i32 2, i32 3>
-; IC4VF4-NEXT: [[VEC_IV8:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 4, i32 5, i32 6, i32 7>
-; IC4VF4-NEXT: [[VEC_IV11:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 8, i32 9, i32 10, i32 11>
-; IC4VF4-NEXT: [[VEC_IV14:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 12, i32 13, i32 14, i32 15>
-; IC4VF4-NEXT: [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], splat (i32 11)
-; IC4VF4-NEXT: [[TMP1:%.*]] = icmp ule <4 x i32> [[VEC_IV8]], splat (i32 11)
-; IC4VF4-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV11]], splat (i32 11)
-; IC4VF4-NEXT: [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV14]], splat (i32 11)
-; IC4VF4-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
-; IC4VF4-NEXT: br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; IC4VF4: [[PRED_LOAD_IF]]:
-; IC4VF4-NEXT: [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 0
-; IC4VF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP5]]
+; IC4VF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 12
; IC4VF4-NEXT: [[TMP7:%.*]] = load i16, ptr [[TMP6]], align 1
; IC4VF4-NEXT: [[TMP8:%.*]] = insertelement <4 x i16> poison, i16 [[TMP7]], i32 0
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; IC4VF4: [[PRED_LOAD_CONTINUE]]:
; IC4VF4-NEXT: [[TMP9:%.*]] = phi <4 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
-; IC4VF4-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
-; IC4VF4-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
; IC4VF4: [[PRED_LOAD_IF15]]:
-; IC4VF4-NEXT: [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], -1
-; IC4VF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP11]]
+; IC4VF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 11
; IC4VF4-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP12]], align 1
; IC4VF4-NEXT: [[TMP14:%.*]] = insertelement <4 x i16> [[TMP9]], i16 [[TMP13]], i32 1
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
; IC4VF4: [[PRED_LOAD_CONTINUE16]]:
; IC4VF4-NEXT: [[TMP15:%.*]] = phi <4 x i16> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF15]] ]
-; IC4VF4-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
-; IC4VF4-NEXT: br i1 [[TMP16]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
; IC4VF4: [[PRED_LOAD_IF17]]:
-; IC4VF4-NEXT: [[TMP17:%.*]] = add i16 [[OFFSET_IDX]], -2
-; IC4VF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP17]]
+; IC4VF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 10
; IC4VF4-NEXT: [[TMP19:%.*]] = load i16, ptr [[TMP18]], align 1
; IC4VF4-NEXT: [[TMP20:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP19]], i32 2
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE18]]
; IC4VF4: [[PRED_LOAD_CONTINUE18]]:
; IC4VF4-NEXT: [[TMP21:%.*]] = phi <4 x i16> [ [[TMP15]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP20]], %[[PRED_LOAD_IF17]] ]
-; IC4VF4-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
-; IC4VF4-NEXT: br i1 [[TMP22]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
; IC4VF4: [[PRED_LOAD_IF19]]:
-; IC4VF4-NEXT: [[TMP23:%.*]] = add i16 [[OFFSET_IDX]], -3
-; IC4VF4-NEXT: [[TMP24:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP23]]
+; IC4VF4-NEXT: [[TMP24:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 9
; IC4VF4-NEXT: [[TMP25:%.*]] = load i16, ptr [[TMP24]], align 1
; IC4VF4-NEXT: [[TMP26:%.*]] = insertelement <4 x i16> [[TMP21]], i16 [[TMP25]], i32 3
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE20]]
; IC4VF4: [[PRED_LOAD_CONTINUE20]]:
; IC4VF4-NEXT: [[TMP27:%.*]] = phi <4 x i16> [ [[TMP21]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP26]], %[[PRED_LOAD_IF19]] ]
-; IC4VF4-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
-; IC4VF4-NEXT: br i1 [[TMP28]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
; IC4VF4: [[PRED_LOAD_IF21]]:
-; IC4VF4-NEXT: [[TMP29:%.*]] = add i16 [[OFFSET_IDX]], -4
-; IC4VF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP29]]
+; IC4VF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 8
; IC4VF4-NEXT: [[TMP31:%.*]] = load i16, ptr [[TMP30]], align 1
; IC4VF4-NEXT: [[TMP32:%.*]] = insertelement <4 x i16> poison, i16 [[TMP31]], i32 0
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE22]]
; IC4VF4: [[PRED_LOAD_CONTINUE22]]:
; IC4VF4-NEXT: [[TMP33:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP32]], %[[PRED_LOAD_IF21]] ]
-; IC4VF4-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
-; IC4VF4-NEXT: br i1 [[TMP34]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
; IC4VF4: [[PRED_LOAD_IF23]]:
-; IC4VF4-NEXT: [[TMP35:%.*]] = add i16 [[OFFSET_IDX]], -5
-; IC4VF4-NEXT: [[TMP36:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP35]]
+; IC4VF4-NEXT: [[TMP36:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 7
; IC4VF4-NEXT: [[TMP37:%.*]] = load i16, ptr [[TMP36]], align 1
; IC4VF4-NEXT: [[TMP38:%.*]] = insertelement <4 x i16> [[TMP33]], i16 [[TMP37]], i32 1
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE24]]
; IC4VF4: [[PRED_LOAD_CONTINUE24]]:
; IC4VF4-NEXT: [[TMP39:%.*]] = phi <4 x i16> [ [[TMP33]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP38]], %[[PRED_LOAD_IF23]] ]
-; IC4VF4-NEXT: [[TMP40:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
-; IC4VF4-NEXT: br i1 [[TMP40]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
; IC4VF4: [[PRED_LOAD_IF25]]:
-; IC4VF4-NEXT: [[TMP41:%.*]] = add i16 [[OFFSET_IDX]], -6
-; IC4VF4-NEXT: [[TMP42:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP41]]
+; IC4VF4-NEXT: [[TMP42:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 6
; IC4VF4-NEXT: [[TMP43:%.*]] = load i16, ptr [[TMP42]], align 1
; IC4VF4-NEXT: [[TMP44:%.*]] = insertelement <4 x i16> [[TMP39]], i16 [[TMP43]], i32 2
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE26]]
; IC4VF4: [[PRED_LOAD_CONTINUE26]]:
; IC4VF4-NEXT: [[TMP45:%.*]] = phi <4 x i16> [ [[TMP39]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP44]], %[[PRED_LOAD_IF25]] ]
-; IC4VF4-NEXT: [[TMP46:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
-; IC4VF4-NEXT: br i1 [[TMP46]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
; IC4VF4: [[PRED_LOAD_IF27]]:
-; IC4VF4-NEXT: [[TMP47:%.*]] = add i16 [[OFFSET_IDX]], -7
-; IC4VF4-NEXT: [[TMP48:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP47]]
+; IC4VF4-NEXT: [[TMP48:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 5
; IC4VF4-NEXT: [[TMP49:%.*]] = load i16, ptr [[TMP48]], align 1
; IC4VF4-NEXT: [[TMP50:%.*]] = insertelement <4 x i16> [[TMP45]], i16 [[TMP49]], i32 3
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE28]]
; IC4VF4: [[PRED_LOAD_CONTINUE28]]:
; IC4VF4-NEXT: [[TMP51:%.*]] = phi <4 x i16> [ [[TMP45]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP50]], %[[PRED_LOAD_IF27]] ]
-; IC4VF4-NEXT: [[TMP52:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
-; IC4VF4-NEXT: br i1 [[TMP52]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
; IC4VF4: [[PRED_LOAD_IF29]]:
-; IC4VF4-NEXT: [[TMP53:%.*]] = add i16 [[OFFSET_IDX]], -8
-; IC4VF4-NEXT: [[TMP54:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP53]]
+; IC4VF4-NEXT: [[TMP54:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 4
; IC4VF4-NEXT: [[TMP55:%.*]] = load i16, ptr [[TMP54]], align 1
; IC4VF4-NEXT: [[TMP56:%.*]] = insertelement <4 x i16> poison, i16 [[TMP55]], i32 0
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE30]]
; IC4VF4: [[PRED_LOAD_CONTINUE30]]:
; IC4VF4-NEXT: [[TMP57:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP56]], %[[PRED_LOAD_IF29]] ]
-; IC4VF4-NEXT: [[TMP58:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
-; IC4VF4-NEXT: br i1 [[TMP58]], label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
; IC4VF4: [[PRED_LOAD_IF31]]:
-; IC4VF4-NEXT: [[TMP59:%.*]] = add i16 [[OFFSET_IDX]], -9
-; IC4VF4-NEXT: [[TMP60:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP59]]
+; IC4VF4-NEXT: [[TMP60:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 3
; IC4VF4-NEXT: [[TMP61:%.*]] = load i16, ptr [[TMP60]], align 1
; IC4VF4-NEXT: [[TMP62:%.*]] = insertelement <4 x i16> [[TMP57]], i16 [[TMP61]], i32 1
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE32]]
; IC4VF4: [[PRED_LOAD_CONTINUE32]]:
; IC4VF4-NEXT: [[TMP63:%.*]] = phi <4 x i16> [ [[TMP57]], %[[PRED_LOAD_CONTINUE30]] ], [ [[TMP62]], %[[PRED_LOAD_IF31]] ]
-; IC4VF4-NEXT: [[TMP64:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
-; IC4VF4-NEXT: br i1 [[TMP64]], label %[[PRED_LOAD_IF33:.*]], label %[[PRED_LOAD_CONTINUE34:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF33:.*]], label %[[PRED_LOAD_CONTINUE34:.*]]
; IC4VF4: [[PRED_LOAD_IF33]]:
-; IC4VF4-NEXT: [[TMP65:%.*]] = add i16 [[OFFSET_IDX]], -10
-; IC4VF4-NEXT: [[TMP66:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP65]]
+; IC4VF4-NEXT: [[TMP66:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 2
; IC4VF4-NEXT: [[TMP67:%.*]] = load i16, ptr [[TMP66]], align 1
; IC4VF4-NEXT: [[TMP68:%.*]] = insertelement <4 x i16> [[TMP63]], i16 [[TMP67]], i32 2
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE34]]
; IC4VF4: [[PRED_LOAD_CONTINUE34]]:
; IC4VF4-NEXT: [[TMP69:%.*]] = phi <4 x i16> [ [[TMP63]], %[[PRED_LOAD_CONTINUE32]] ], [ [[TMP68]], %[[PRED_LOAD_IF33]] ]
-; IC4VF4-NEXT: [[TMP70:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
-; IC4VF4-NEXT: br i1 [[TMP70]], label %[[PRED_LOAD_IF35:.*]], label %[[PRED_LOAD_CONTINUE36:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF35:.*]], label %[[PRED_LOAD_CONTINUE36:.*]]
; IC4VF4: [[PRED_LOAD_IF35]]:
-; IC4VF4-NEXT: [[TMP71:%.*]] = add i16 [[OFFSET_IDX]], -11
-; IC4VF4-NEXT: [[TMP72:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP71]]
+; IC4VF4-NEXT: [[TMP72:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 1
; IC4VF4-NEXT: [[TMP73:%.*]] = load i16, ptr [[TMP72]], align 1
; IC4VF4-NEXT: [[TMP74:%.*]] = insertelement <4 x i16> [[TMP69]], i16 [[TMP73]], i32 3
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE36]]
; IC4VF4: [[PRED_LOAD_CONTINUE36]]:
; IC4VF4-NEXT: [[TMP75:%.*]] = phi <4 x i16> [ [[TMP69]], %[[PRED_LOAD_CONTINUE34]] ], [ [[TMP74]], %[[PRED_LOAD_IF35]] ]
-; IC4VF4-NEXT: [[TMP76:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
-; IC4VF4-NEXT: br i1 [[TMP76]], label %[[PRED_LOAD_IF37:.*]], label %[[PRED_LOAD_CONTINUE38:.*]]
+; IC4VF4-NEXT: br i1 false, label %[[PRED_LOAD_IF37:.*]], label %[[PRED_LOAD_CONTINUE38:.*]]
; IC4VF4: [[PRED_LOAD_IF37]]:
-; IC4VF4-NEXT: [[TMP77:%.*]] = add i16 [[OFFSET_IDX]], -12
-; IC4VF4-NEXT: [[TMP78:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP77]]
+; IC4VF4-NEXT: [[TMP78:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 0
; IC4VF4-NEXT: [[TMP79:%.*]] = load i16, ptr [[TMP78]], align 1
; IC4VF4-NEXT: [[TMP80:%.*]] = insertelement <4 x i16> poison, i16 [[TMP79]], i32 0
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE38]]
; IC4VF4: [[PRED_LOAD_CONTINUE38]]:
; IC4VF4-NEXT: [[TMP81:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE36]] ], [ [[TMP80]], %[[PRED_LOAD_IF37]] ]
-; IC4VF4-NEXT: [[TMP82:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
-; IC4VF4-NEXT: br i1 [[TMP82]], label %[[PRED_LOAD_IF39:.*]], label %[[PRED_LOAD_CONTINUE40:.*]]
+; IC4VF4-NEXT: br i1 false, label %[[PRED_LOAD_IF39:.*]], label %[[PRED_LOAD_CONTINUE40:.*]]
; IC4VF4: [[PRED_LOAD_IF39]]:
-; IC4VF4-NEXT: [[TMP83:%.*]] = add i16 [[OFFSET_IDX]], -13
-; IC4VF4-NEXT: [[TMP84:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP83]]
+; IC4VF4-NEXT: [[TMP84:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 -1
; IC4VF4-NEXT: [[TMP85:%.*]] = load i16, ptr [[TMP84]], align 1
; IC4VF4-NEXT: [[TMP86:%.*]] = insertelement <4 x i16> [[TMP81]], i16 [[TMP85]], i32 1
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE40]]
; IC4VF4: [[PRED_LOAD_CONTINUE40]]:
; IC4VF4-NEXT: [[TMP87:%.*]] = phi <4 x i16> [ [[TMP81]], %[[PRED_LOAD_CONTINUE38]] ], [ [[TMP86]], %[[PRED_LOAD_IF39]] ]
-; IC4VF4-NEXT: [[TMP88:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
-; IC4VF4-NEXT: br i1 [[TMP88]], label %[[PRED_LOAD_IF41:.*]], label %[[PRED_LOAD_CONTINUE42:.*]]
+; IC4VF4-NEXT: br i1 false, label %[[PRED_LOAD_IF41:.*]], label %[[PRED_LOAD_CONTINUE42:.*]]
; IC4VF4: [[PRED_LOAD_IF41]]:
-; IC4VF4-NEXT: [[TMP89:%.*]] = add i16 [[OFFSET_IDX]], -14
-; IC4VF4-NEXT: [[TMP90:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP89]]
+; IC4VF4-NEXT: [[TMP90:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 -2
; IC4VF4-NEXT: [[TMP91:%.*]] = load i16, ptr [[TMP90]], align 1
; IC4VF4-NEXT: [[TMP92:%.*]] = insertelement <4 x i16> [[TMP87]], i16 [[TMP91]], i32 2
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE42]]
; IC4VF4: [[PRED_LOAD_CONTINUE42]]:
; IC4VF4-NEXT: [[TMP93:%.*]] = phi <4 x i16> [ [[TMP87]], %[[PRED_LOAD_CONTINUE40]] ], [ [[TMP92]], %[[PRED_LOAD_IF41]] ]
-; IC4VF4-NEXT: [[TMP94:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
-; IC4VF4-NEXT: br i1 [[TMP94]], label %[[PRED_LOAD_IF43:.*]], label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4-NEXT: br i1 false, label %[[PRED_LOAD_IF43:.*]], label %[[PRED_LOAD_CONTINUE44:.*]]
; IC4VF4: [[PRED_LOAD_IF43]]:
-; IC4VF4-NEXT: [[TMP95:%.*]] = add i16 [[OFFSET_IDX]], -15
-; IC4VF4-NEXT: [[TMP96:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP95]]
+; IC4VF4-NEXT: [[TMP96:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 -3
; IC4VF4-NEXT: [[TMP97:%.*]] = load i16, ptr [[TMP96]], align 1
; IC4VF4-NEXT: [[TMP98:%.*]] = insertelement <4 x i16> [[TMP93]], i16 [[TMP97]], i32 3
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE44]]
@@ -379,22 +326,16 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
; IC4VF4-NEXT: [[TMP101:%.*]] = icmp ugt <4 x i16> [[TMP51]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP102:%.*]] = icmp ugt <4 x i16> [[TMP75]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP103:%.*]] = icmp ugt <4 x i16> [[TMP99]], [[BROADCAST_SPLAT]]
-; IC4VF4-NEXT: [[TMP104:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
-; IC4VF4-NEXT: [[TMP105:%.*]] = add nsw <4 x i16> [[STEP_ADD]], splat (i16 -1)
-; IC4VF4-NEXT: [[TMP106:%.*]] = add nsw <4 x i16> [[STEP_ADD_2]], splat (i16 -1)
-; IC4VF4-NEXT: [[TMP107:%.*]] = add nsw <4 x i16> [[STEP_ADD_3]], splat (i16 -1)
-; IC4VF4-NEXT: [[TMP108]] = select <4 x i1> [[TMP100]], <4 x i16> [[TMP104]], <4 x i16> [[VEC_PHI]]
-; IC4VF4-NEXT: [[TMP109]] = select <4 x i1> [[TMP101]], <4 x i16> [[TMP105]], <4 x i16> [[VEC_PHI1]]
-; IC4VF4-NEXT: [[TMP110]] = select <4 x i1> [[TMP102]], <4 x i16> [[TMP106]], <4 x i16> [[VEC_PHI2]]
-; IC4VF4-NEXT: [[TMP111]] = select <4 x i1> [[TMP103]], <4 x i16> [[TMP107]], <4 x i16> [[VEC_PHI3]]
-; IC4VF4-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; IC4VF4-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i16> [[STEP_ADD_3]], splat (i16 -4)
-; IC4VF4-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> [[TMP100]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> [[TMP101]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> [[TMP102]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[TMP112:%.*]] = select <4 x i1> [[TMP0]], <4 x i16> [[TMP108]], <4 x i16> [[VEC_PHI]]
-; IC4VF4-NEXT: [[TMP113:%.*]] = select <4 x i1> [[TMP1]], <4 x i16> [[TMP109]], <4 x i16> [[VEC_PHI1]]
-; IC4VF4-NEXT: [[TMP114:%.*]] = select <4 x i1> [[TMP2]], <4 x i16> [[TMP110]], <4 x i16> [[VEC_PHI2]]
-; IC4VF4-NEXT: [[TMP115:%.*]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP111]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT: [[TMP112:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP76]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP113:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP77]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP114:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP70]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP115:%.*]] = select <4 x i1> zeroinitializer, <4 x i16> [[TMP71]], <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP112]], <4 x i16> [[TMP113]])
; IC4VF4-NEXT: [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP114]])
; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP115]])
@@ -519,182 +460,129 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
; IC4VF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x half> [[BROADCAST_SPLATINSERT]], <4 x half> poison, <4 x i32> zeroinitializer
; IC4VF4-NEXT: br label %[[VECTOR_BODY:.*]]
; IC4VF4: [[VECTOR_BODY]]:
-; IC4VF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44:.*]] ]
-; IC4VF4-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP108:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP109:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP110:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP111:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
-; IC4VF4-NEXT: [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
-; IC4VF4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 -4)
-; IC4VF4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i16> [[STEP_ADD_2]], splat (i16 -4)
-; IC4VF4-NEXT: [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
-; IC4VF4-NEXT: [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
-; IC4VF4-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
-; IC4VF4-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
-; IC4VF4-NEXT: [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 0, i32 1, i32 2, i32 3>
-; IC4VF4-NEXT: [[VEC_IV8:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 4, i32 5, i32 6, i32 7>
-; IC4VF4-NEXT: [[VEC_IV11:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 8, i32 9, i32 10, i32 11>
-; IC4VF4-NEXT: [[VEC_IV14:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 12, i32 13, i32 14, i32 15>
-; IC4VF4-NEXT: [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], splat (i32 11)
-; IC4VF4-NEXT: [[TMP1:%.*]] = icmp ule <4 x i32> [[VEC_IV8]], splat (i32 11)
-; IC4VF4-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV11]], splat (i32 11)
-; IC4VF4-NEXT: [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV14]], splat (i32 11)
-; IC4VF4-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
-; IC4VF4-NEXT: br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; IC4VF4: [[PRED_LOAD_IF]]:
-; IC4VF4-NEXT: [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 0
-; IC4VF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP5]]
+; IC4VF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 12
; IC4VF4-NEXT: [[TMP7:%.*]] = load half, ptr [[TMP6]], align 1
; IC4VF4-NEXT: [[TMP8:%.*]] = insertelement <4 x half> poison, half [[TMP7]], i32 0
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; IC4VF4: [[PRED_LOAD_CONTINUE]]:
; IC4VF4-NEXT: [[TMP9:%.*]] = phi <4 x half> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
-; IC4VF4-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
-; IC4VF4-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
; IC4VF4: [[PRED_LOAD_IF15]]:
-; IC4VF4-NEXT: [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], -1
-; IC4VF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP11]]
+; IC4VF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 11
; IC4VF4-NEXT: [[TMP13:%.*]] = load half, ptr [[TMP12]], align 1
; IC4VF4-NEXT: [[TMP14:%.*]] = insertelement <4 x half> [[TMP9]], half [[TMP13]], i32 1
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
; IC4VF4: [[PRED_LOAD_CONTINUE16]]:
; IC4VF4-NEXT: [[TMP15:%.*]] = phi <4 x half> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF15]] ]
-; IC4VF4-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
-; IC4VF4-NEXT: br i1 [[TMP16]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
; IC4VF4: [[PRED_LOAD_IF17]]:
-; IC4VF4-NEXT: [[TMP17:%.*]] = add i16 [[OFFSET_IDX]], -2
-; IC4VF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP17]]
+; IC4VF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 10
; IC4VF4-NEXT: [[TMP19:%.*]] = load half, ptr [[TMP18]], align 1
; IC4VF4-NEXT: [[TMP20:%.*]] = insertelement <4 x half> [[TMP15]], half [[TMP19]], i32 2
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE18]]
; IC4VF4: [[PRED_LOAD_CONTINUE18]]:
; IC4VF4-NEXT: [[TMP21:%.*]] = phi <4 x half> [ [[TMP15]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP20]], %[[PRED_LOAD_IF17]] ]
-; IC4VF4-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
-; IC4VF4-NEXT: br i1 [[TMP22]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
; IC4VF4: [[PRED_LOAD_IF19]]:
-; IC4VF4-NEXT: [[TMP23:%.*]] = add i16 [[OFFSET_IDX]], -3
-; IC4VF4-NEXT: [[TMP24:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP23]]
+; IC4VF4-NEXT: [[TMP24:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 9
; IC4VF4-NEXT: [[TMP25:%.*]] = load half, ptr [[TMP24]], align 1
; IC4VF4-NEXT: [[TMP26:%.*]] = insertelement <4 x half> [[TMP21]], half [[TMP25]], i32 3
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE20]]
; IC4VF4: [[PRED_LOAD_CONTINUE20]]:
; IC4VF4-NEXT: [[TMP27:%.*]] = phi <4 x half> [ [[TMP21]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP26]], %[[PRED_LOAD_IF19]] ]
-; IC4VF4-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
-; IC4VF4-NEXT: br i1 [[TMP28]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
; IC4VF4: [[PRED_LOAD_IF21]]:
-; IC4VF4-NEXT: [[TMP29:%.*]] = add i16 [[OFFSET_IDX]], -4
-; IC4VF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP29]]
+; IC4VF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 8
; IC4VF4-NEXT: [[TMP31:%.*]] = load half, ptr [[TMP30]], align 1
; IC4VF4-NEXT: [[TMP32:%.*]] = insertelement <4 x half> poison, half [[TMP31]], i32 0
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE22]]
; IC4VF4: [[PRED_LOAD_CONTINUE22]]:
; IC4VF4-NEXT: [[TMP33:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP32]], %[[PRED_LOAD_IF21]] ]
-; IC4VF4-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
-; IC4VF4-NEXT: br i1 [[TMP34]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
; IC4VF4: [[PRED_LOAD_IF23]]:
-; IC4VF4-NEXT: [[TMP35:%.*]] = add i16 [[OFFSET_IDX]], -5
-; IC4VF4-NEXT: [[TMP36:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP35]]
+; IC4VF4-NEXT: [[TMP36:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 7
; IC4VF4-NEXT: [[TMP37:%.*]] = load half, ptr [[TMP36]], align 1
; IC4VF4-NEXT: [[TMP38:%.*]] = insertelement <4 x half> [[TMP33]], half [[TMP37]], i32 1
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE24]]
; IC4VF4: [[PRED_LOAD_CONTINUE24]]:
; IC4VF4-NEXT: [[TMP39:%.*]] = phi <4 x half> [ [[TMP33]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP38]], %[[PRED_LOAD_IF23]] ]
-; IC4VF4-NEXT: [[TMP40:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
-; IC4VF4-NEXT: br i1 [[TMP40]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
; IC4VF4: [[PRED_LOAD_IF25]]:
-; IC4VF4-NEXT: [[TMP41:%.*]] = add i16 [[OFFSET_IDX]], -6
-; IC4VF4-NEXT: [[TMP42:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP41]]
+; IC4VF4-NEXT: [[TMP42:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 6
; IC4VF4-NEXT: [[TMP43:%.*]] = load half, ptr [[TMP42]], align 1
; IC4VF4-NEXT: [[TMP44:%.*]] = insertelement <4 x half> [[TMP39]], half [[TMP43]], i32 2
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE26]]
; IC4VF4: [[PRED_LOAD_CONTINUE26]]:
; IC4VF4-NEXT: [[TMP45:%.*]] = phi <4 x half> [ [[TMP39]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP44]], %[[PRED_LOAD_IF25]] ]
-; IC4VF4-NEXT: [[TMP46:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
-; IC4VF4-NEXT: br i1 [[TMP46]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
; IC4VF4: [[PRED_LOAD_IF27]]:
-; IC4VF4-NEXT: [[TMP47:%.*]] = add i16 [[OFFSET_IDX]], -7
-; IC4VF4-NEXT: [[TMP48:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP47]]
+; IC4VF4-NEXT: [[TMP48:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 5
; IC4VF4-NEXT: [[TMP49:%.*]] = load half, ptr [[TMP48]], align 1
; IC4VF4-NEXT: [[TMP50:%.*]] = insertelement <4 x half> [[TMP45]], half [[TMP49]], i32 3
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE28]]
; IC4VF4: [[PRED_LOAD_CONTINUE28]]:
; IC4VF4-NEXT: [[TMP51:%.*]] = phi <4 x half> [ [[TMP45]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP50]], %[[PRED_LOAD_IF27]] ]
-; IC4VF4-NEXT: [[TMP52:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
-; IC4VF4-NEXT: br i1 [[TMP52]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
; IC4VF4: [[PRED_LOAD_IF29]]:
-; IC4VF4-NEXT: [[TMP53:%.*]] = add i16 [[OFFSET_IDX]], -8
-; IC4VF4-NEXT: [[TMP54:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP53]]
+; IC4VF4-NEXT: [[TMP54:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 4
; IC4VF4-NEXT: [[TMP55:%.*]] = load half, ptr [[TMP54]], align 1
; IC4VF4-NEXT: [[TMP56:%.*]] = insertelement <4 x half> poison, half [[TMP55]], i32 0
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE30]]
; IC4VF4: [[PRED_LOAD_CONTINUE30]]:
; IC4VF4-NEXT: [[TMP57:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP56]], %[[PRED_LOAD_IF29]] ]
-; IC4VF4-NEXT: [[TMP58:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
-; IC4VF4-NEXT: br i1 [[TMP58]], label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
; IC4VF4: [[PRED_LOAD_IF31]]:
-; IC4VF4-NEXT: [[TMP59:%.*]] = add i16 [[OFFSET_IDX]], -9
-; IC4VF4-NEXT: [[TMP60:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP59]]
+; IC4VF4-NEXT: [[TMP60:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 3
; IC4VF4-NEXT: [[TMP61:%.*]] = load half, ptr [[TMP60]], align 1
; IC4VF4-NEXT: [[TMP62:%.*]] = insertelement <4 x half> [[TMP57]], half [[TMP61]], i32 1
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE32]]
; IC4VF4: [[PRED_LOAD_CONTINUE32]]:
; IC4VF4-NEXT: [[TMP63:%.*]] = phi <4 x half> [ [[TMP57]], %[[PRED_LOAD_CONTINUE30]] ], [ [[TMP62]], %[[PRED_LOAD_IF31]] ]
-; IC4VF4-NEXT: [[TMP64:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
-; IC4VF4-NEXT: br i1 [[TMP64]], label %[[PRED_LOAD_IF33:.*]], label %[[PRED_LOAD_CONTINUE34:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF33:.*]], label %[[PRED_LOAD_CONTINUE34:.*]]
; IC4VF4: [[PRED_LOAD_IF33]]:
-; IC4VF4-NEXT: [[TMP65:%.*]] = add i16 [[OFFSET_IDX]], -10
-; IC4VF4-NEXT: [[TMP66:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP65]]
+; IC4VF4-NEXT: [[TMP66:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 2
; IC4VF4-NEXT: [[TMP67:%.*]] = load half, ptr [[TMP66]], align 1
; IC4VF4-NEXT: [[TMP68:%.*]] = insertelement <4 x half> [[TMP63]], half [[TMP67]], i32 2
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE34]]
; IC4VF4: [[PRED_LOAD_CONTINUE34]]:
; IC4VF4-NEXT: [[TMP69:%.*]] = phi <4 x half> [ [[TMP63]], %[[PRED_LOAD_CONTINUE32]] ], [ [[TMP68]], %[[PRED_LOAD_IF33]] ]
-; IC4VF4-NEXT: [[TMP70:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
-; IC4VF4-NEXT: br i1 [[TMP70]], label %[[PRED_LOAD_IF35:.*]], label %[[PRED_LOAD_CONTINUE36:.*]]
+; IC4VF4-NEXT: br i1 true, label %[[PRED_LOAD_IF35:.*]], label %[[PRED_LOAD_CONTINUE36:.*]]
; IC4VF4: [[PRED_LOAD_IF35]]:
-; IC4VF4-NEXT: [[TMP71:%.*]] = add i16 [[OFFSET_IDX]], -11
-; IC4VF4-NEXT: [[TMP72:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP71]]
+; IC4VF4-NEXT: [[TMP72:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 1
; IC4VF4-NEXT: [[TMP73:%.*]] = load half, ptr [[TMP72]], align 1
; IC4VF4-NEXT: [[TMP74:%.*]] = insertelement <4 x half> [[TMP69]], half [[TMP73]], i32 3
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE36]]
; IC4VF4: [[PRED_LOAD_CONTINUE36]]:
; IC4VF4-NEXT: [[TMP75:%.*]] = phi <4 x half> [ [[TMP69]], %[[PRED_LOAD_CONTINUE34]] ], [ [[TMP74]], %[[PRED_LOAD_IF35]] ]
-; IC4VF4-NEXT: [[TMP76:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
-; IC4VF4-NEXT: br i1 [[TMP76]], label %[[PRED_LOAD_IF37:.*]], label %[[PRED_LOAD_CONTINUE38:.*]]
+; IC4VF4-NEXT: br i1 false, label %[[PRED_LOAD_IF37:.*]], label %[[PRED_LOAD_CONTINUE38:.*]]
; IC4VF4: [[PRED_LOAD_IF37]]:
-; IC4VF4-NEXT: [[TMP77:%.*]] = add i16 [[OFFSET_IDX]], -12
-; IC4VF4-NEXT: [[TMP78:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP77]]
+; IC4VF4-NEXT: [[TMP78:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 0
; IC4VF4-NEXT: [[TMP79:%.*]] = load half, ptr [[TMP78]], align 1
; IC4VF4-NEXT: [[TMP80:%.*]] = insertelement <4 x half> poison, half [[TMP79]], i32 0
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE38]]
; IC4VF4: [[PRED_LOAD_CONTINUE38]]:
; IC4VF4-NEXT: [[TMP81:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE36]] ], [ [[TMP80]], %[[PRED_LOAD_IF37]] ]
-; IC4VF4-NEXT: [[TMP82:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
-; IC4VF4-NEXT: br i1 [[TMP82]], label %[[PRED_LOAD_IF39:.*]], label %[[PRED_LOAD_CONTINUE40:.*]]
+; IC4VF4-NEXT: br i1 false, label %[[PRED_LOAD_IF39:.*]], label %[[PRED_LOAD_CONTINUE40:.*]]
; IC4VF4: [[PRED_LOAD_IF39]]:
-; IC4VF4-NEXT: [[TMP83:%.*]] = add i16 [[OFFSET_IDX]], -13
-; IC4VF4-NEXT: [[TMP84:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP83]]
+; IC4VF4-NEXT: [[TMP84:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 -1
; IC4VF4-NEXT: [[TMP85:%.*]] = load half, ptr [[TMP84]], align 1
; IC4VF4-NEXT: [[TMP86:%.*]] = insertelement <4 x half> [[TMP81]], half [[TMP85]], i32 1
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE40]]
; IC4VF4: [[PRED_LOAD_CONTINUE40]]:
; IC4VF4-NEXT: [[TMP87:%.*]] = phi <4 x half> [ [[TMP81]], %[[PRED_LOAD_CONTINUE38]] ], [ [[TMP86]], %[[PRED_LOAD_IF39]] ]
-; IC4VF4-NEXT: [[TMP88:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
-; IC4VF4-NEXT: br i1 [[TMP88]], label %[[PRED_LOAD_IF41:.*]], label %[[PRED_LOAD_CONTINUE42:.*]]
+; IC4VF4-NEXT: br i1 false, label %[[PRED_LOAD_IF41:.*]], label %[[PRED_LOAD_CONTINUE42:.*]]
; IC4VF4: [[PRED_LOAD_IF41]]:
-; IC4VF4-NEXT: [[TMP89:%.*]] = add i16 [[OFFSET_IDX]], -14
-; IC4VF4-NEXT: [[TMP90:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP89]]
+; IC4VF4-NEXT: [[TMP90:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 -2
; IC4VF4-NEXT: [[TMP91:%.*]] = load half, ptr [[TMP90]], align 1
; IC4VF4-NEXT: [[TMP92:%.*]] = insertelement <4 x half> [[TMP87]], half [[TMP91]], i32 2
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE42]]
; IC4VF4: [[PRED_LOAD_CONTINUE42]]:
; IC4VF4-NEXT: [[TMP93:%.*]] = phi <4 x half> [ [[TMP87]], %[[PRED_LOAD_CONTINUE40]] ], [ [[TMP92]], %[[PRED_LOAD_IF41]] ]
-; IC4VF4-NEXT: [[TMP94:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
-; IC4VF4-NEXT: br i1 [[TMP94]], label %[[PRED_LOAD_IF43:.*]], label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4-NEXT: br i1 false, label %[[PRED_LOAD_IF43:.*]], label %[[PRED_LOAD_CONTINUE44:.*]]
; IC4VF4: [[PRED_LOAD_IF43]]:
-; IC4VF4-NEXT: [[TMP95:%.*]] = add i16 [[OFFSET_IDX]], -15
-; IC4VF4-NEXT: [[TMP96:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP95]]
+; IC4VF4-NEXT: [[TMP96:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 -3
; IC4VF4-NEXT: [[TMP97:%.*]] = load half, ptr [[TMP96]], align 1
; IC4VF4-NEXT: [[TMP98:%.*]] = insertelement <4 x half> [[TMP93]], half [[TMP97]], i32 3
; IC4VF4-NEXT: br label %[[PRED_LOAD_CONTINUE44]]
@@ -704,22 +592,16 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
; IC4VF4-NEXT: [[TMP101:%.*]] = fcmp ugt <4 x half> [[TMP51]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP102:%.*]] = fcmp ugt <4 x half> [[TMP75]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP103:%.*]] = fcmp ugt <4 x half> [[TMP99]], [[BROADCAST_SPLAT]]
-; IC4VF4-NEXT: [[TMP104:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
-; IC4VF4-NEXT: [[TMP105:%.*]] = add nsw <4 x i16> [[STEP_ADD]], splat (i16 -1)
-; IC4VF4-NEXT: [[TMP106:%.*]] = add nsw <4 x i16> [[STEP_ADD_2]], splat (i16 -1)
-; IC4VF4-NEXT: [[TMP107:%.*]] = add nsw <4 x i16> [[STEP_ADD_3]], splat (i16 -1)
-; IC4VF4-NEXT: [[TMP108]] = select <4 x i1> [[TMP100]], <4 x i16> [[TMP104]], <4 x i16> [[VEC_PHI]]
-; IC4VF4-NEXT: [[TMP109]] = select <4 x i1> [[TMP101]], <4 x i16> [[TMP105]], <4 x i16> [[VEC_PHI1]]
-; IC4VF4-NEXT: [[TMP110]] = select <4 x i1> [[TMP102]], <4 x i16> [[TMP106]], <4 x i16> [[VEC_PHI2]]
-; IC4VF4-NEXT: [[TMP111]] = select <4 x i1> [[TMP103]], <4 x i16> [[TMP107]], <4 x i16> [[VEC_PHI3]]
-; IC4VF4-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; IC4VF4-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i16> [[STEP_ADD_3]], splat (i16 -4)
-; IC4VF4-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> [[TMP100]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> [[TMP101]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> [[TMP102]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[TMP112:%.*]] = select <4 x i1> [[TMP0]], <4 x i16> [[TMP108]], <4 x i16> [[VEC_PHI]]
-; IC4VF4-NEXT: [[TMP113:%.*]] = select <4 x i1> [[TMP1]], <4 x i16> [[TMP109]], <4 x i16> [[VEC_PHI1]]
-; IC4VF4-NEXT: [[TMP114:%.*]] = select <4 x i1> [[TMP2]], <4 x i16> [[TMP110]], <4 x i16> [[VEC_PHI2]]
-; IC4VF4-NEXT: [[TMP115:%.*]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP111]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT: [[TMP112:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP76]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP113:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP77]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP114:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP70]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP115:%.*]] = select <4 x i1> zeroinitializer, <4 x i16> [[TMP71]], <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP112]], <4 x i16> [[TMP113]])
; IC4VF4-NEXT: [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP114]])
; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP115]])
@@ -873,7 +755,7 @@ define i64 @select_decreasing_induction_icmp_iv_unsigned(ptr %a) {
; IC4VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; IC4VF4-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[STEP_ADD_3]], splat (i64 -4)
; IC4VF4-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], -9223372036854775808
-; IC4VF4-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; IC4VF4-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; IC4VF4: [[MIDDLE_BLOCK]]:
; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> [[TMP13]], <4 x i64> [[TMP14]])
; IC4VF4-NEXT: [[RDX_MINMAX10:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> [[RDX_MINMAX]], <4 x i64> [[TMP15]])
@@ -1102,7 +984,7 @@ define i64 @select_decreasing_induction_icmp_iv_just_within_bounds(ptr %a, ptr %
; IC4VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; IC4VF4-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD_3]], splat (i64 -4)
; IC4VF4-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], -16
-; IC4VF4-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IC4VF4-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; IC4VF4: [[MIDDLE_BLOCK]]:
; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> [[TMP27]], <4 x i64> [[TMP28]])
; IC4VF4-NEXT: [[RDX_MINMAX18:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> [[RDX_MINMAX]], <4 x i64> [[TMP29]])
@@ -1124,7 +1006,7 @@ define i64 @select_decreasing_induction_icmp_iv_just_within_bounds(ptr %a, ptr %
; IC4VF4-NEXT: [[CMP_A_B:%.*]] = icmp sgt i8 [[LD_A]], [[LD_B]]
; IC4VF4-NEXT: [[COND]] = select i1 [[CMP_A_B]], i64 [[IV_NEXT]], i64 [[RDX]]
; IC4VF4-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; IC4VF4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; IC4VF4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; IC4VF4: [[EXIT]]:
; IC4VF4-NEXT: [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[LOOP]] ]
; IC4VF4-NEXT: ret i64 [[COND_LCSSA]]
@@ -1363,7 +1245,7 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; IC4VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; IC4VF4-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[STEP_ADD_3]], splat (i64 -4)
; IC4VF4-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IC4VF4-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; IC4VF4-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; IC4VF4: [[MIDDLE_BLOCK]]:
; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[TMP34]], <4 x i64> [[TMP35]])
; IC4VF4-NEXT: [[RDX_MINMAX23:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[RDX_MINMAX]], <4 x i64> [[TMP36]])
@@ -1392,7 +1274,7 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; IC4VF4-NEXT: [[CMP_A_B:%.*]] = icmp sgt i64 [[LD_A]], [[LD_B]]
; IC4VF4-NEXT: [[COND]] = select i1 [[CMP_A_B]], i64 [[IV_NEXT]], i64 [[RDX]]
; IC4VF4-NEXT: [[EXIT_COND:%.*]] = icmp ugt i64 [[IV]], 1
-; IC4VF4-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; IC4VF4-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; IC4VF4: [[EXIT]]:
; IC4VF4-NEXT: [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
; IC4VF4-NEXT: ret i64 [[COND_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
index 5e88072517b37..b5b1879ec6c25 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
@@ -17,58 +17,45 @@ define void @ptr_depends_on_sdiv(ptr noalias %dst, i16 noundef %off) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE4]] ]
-; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK-NEXT: br i1 false, label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP2:%.*]] = sdiv i16 24316, [[OFF]]
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i16> poison, i16 [[TMP2]], i32 0
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
; CHECK: [[PRED_SDIV_CONTINUE]]:
; CHECK-NEXT: [[TMP4:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
-; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
+; CHECK-NEXT: br i1 true, label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
; CHECK: [[PRED_SDIV_IF1]]:
; CHECK-NEXT: [[TMP18:%.*]] = sdiv i16 24316, [[OFF]]
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> [[TMP4]], i16 [[TMP18]], i32 1
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x i16> [ [[TMP4]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP7]], %[[PRED_SDIV_IF1]] ]
-; CHECK-NEXT: [[TMP21:%.*]] = add <2 x i16> [[VEC_IND]], splat (i16 16383)
; CHECK-NEXT: [[TMP22:%.*]] = shl <2 x i16> [[TMP8]], splat (i16 14)
-; CHECK-NEXT: [[TMP23:%.*]] = sub <2 x i16> [[TMP21]], [[TMP22]]
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
-; CHECK-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT: [[TMP23:%.*]] = sub <2 x i16> <i16 16392, i16 16393>, [[TMP22]]
+; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i16> [[TMP23]], i32 0
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP13]]
; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP24]], align 1
-; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[OFFSET_IDX]], 0
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 9
; CHECK-NEXT: store i64 [[TMP9]], ptr [[TMP11]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
-; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF3:.*]], label %[[EXIT:.*]]
; CHECK: [[PRED_STORE_IF3]]:
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <2 x i16> [[TMP23]], i32 1
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP25]]
; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP20]], align 1
-; CHECK-NEXT: [[TMP16:%.*]] = add i16 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 10
; CHECK-NEXT: store i64 [[TMP15]], ptr [[TMP17]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
-; CHECK: [[PRED_STORE_CONTINUE4]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br [[EXIT:label %.*]]
-; CHECK: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop.header
@@ -111,40 +98,30 @@ define void @ptr_depends_on_possibly_poison_value(ptr noalias %dst, i16 %off) {
; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i16> [[TMP0]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
-; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
-; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i16> [[VEC_IND]], [[TMP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i16> <i16 9, i16 10>, [[TMP1]]
+; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i16> [[TMP3]], i32 0
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP5]]
; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP14]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = add i16 [[OFFSET_IDX]], 0
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 9
; CHECK-NEXT: store i64 [[TMP9]], ptr [[TMP8]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
-; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[EXIT:.*]]
; CHECK: [[PRED_STORE_IF1]]:
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i16> [[TMP3]], i32 1
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP15]]
; CHECK-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP16]], align 1
-; CHECK-NEXT: [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 10
; CHECK-NEXT: store i64 [[TMP13]], ptr [[TMP12]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; CHECK: [[PRED_STORE_CONTINUE2]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br [[EXIT:label %.*]]
-; CHECK: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop.header
@@ -183,38 +160,28 @@ define void @ptr_doesnt_depend_on_poison_or_ub(ptr noalias %dst, i16 noundef %of
; CHECK-NEXT: [[TMP1:%.*]] = add i16 [[TMP0]], [[OFF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
-; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
-; CHECK-NEXT: [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add i16 9, [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr @src, i16 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP7:%.*]] = add i16 [[OFFSET_IDX]], 0
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 9
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
; CHECK-NEXT: store i64 [[TMP9]], ptr [[TMP8]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
-; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[EXIT:.*]]
; CHECK: [[PRED_STORE_IF1]]:
-; CHECK-NEXT: [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 10
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
; CHECK-NEXT: store i64 [[TMP13]], ptr [[TMP12]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; CHECK: [[PRED_STORE_CONTINUE2]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br [[EXIT:label %.*]]
-; CHECK: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop.header
@@ -252,45 +219,35 @@ define void @ptr_depends_on_possibly_poison_value_from_load(ptr noalias %dst) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
-; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr @src, align 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
; CHECK-NEXT: [[TMP2:%.*]] = sub <2 x i16> splat (i16 1), [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i16> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i16> [[VEC_IND]], [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
-; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i16> <i16 9, i16 10>, [[TMP3]]
+; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP15]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = add i16 [[OFFSET_IDX]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 9
; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP9]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
-; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[EXIT:.*]]
; CHECK: [[PRED_STORE_IF1]]:
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP16]]
; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP17]], align 1
-; CHECK-NEXT: [[TMP12:%.*]] = add i16 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 10
; CHECK-NEXT: store i64 [[TMP14]], ptr [[TMP13]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; CHECK: [[PRED_STORE_CONTINUE2]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br [[EXIT:label %.*]]
-; CHECK: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop.header
@@ -329,41 +286,31 @@ define void @ptr_depends_on_noundef_load(ptr noalias %dst) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
-; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr @src, align 1, !noundef [[META10:![0-9]+]]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
+; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr @src, align 1, !noundef [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i16 1, [[TMP0]]
; CHECK-NEXT: [[TMP3:%.*]] = add i16 [[TMP2]], [[TMP0]]
-; CHECK-NEXT: [[TMP4:%.*]] = add i16 [[OFFSET_IDX]], [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i16 9, [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr @src, i16 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
-; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP8:%.*]] = add i16 [[OFFSET_IDX]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 9
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP9]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
-; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[EXIT:.*]]
; CHECK: [[PRED_STORE_IF1]]:
-; CHECK-NEXT: [[TMP12:%.*]] = add i16 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 10
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
; CHECK-NEXT: store i64 [[TMP14]], ptr [[TMP13]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; CHECK: [[PRED_STORE_CONTINUE2]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br [[EXIT:label %.*]]
-; CHECK: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop.header
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-minmax-users-and-predicated.ll b/llvm/test/Transforms/LoopVectorize/reduction-minmax-users-and-predicated.ll
index c9cc8060ff498..7119728d493d3 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-minmax-users-and-predicated.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-minmax-users-and-predicated.ll
@@ -48,57 +48,43 @@ define i32 @chained_smax(i32 %x, ptr %src) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 1)
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[VEC_PHI]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[BROADCAST_SPLAT]], <4 x i32> zeroinitializer)
+; CHECK-NEXT: br i1 true, label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [3 x i32], ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [3 x i32], ptr [[SRC]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i32 0
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
-; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK-NEXT: br i1 true, label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
; CHECK: [[PRED_LOAD_IF1]]:
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [3 x i32], ptr [[SRC]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [3 x i32], ptr [[SRC]], i64 1
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i32 1
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
; CHECK: [[PRED_LOAD_CONTINUE2]]:
; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
-; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK-NEXT: br i1 false, label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [3 x i32], ptr [[SRC]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [3 x i32], ptr [[SRC]], i64 2
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i32 2
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
; CHECK: [[PRED_LOAD_CONTINUE4]]:
; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
-; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK-NEXT: br i1 false, label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [3 x i32], ptr [[SRC]], i64 [[TMP21]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [3 x i32], ptr [[SRC]], i64 3
; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i32 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: [[TMP26]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP25]], <4 x i32> [[TMP1]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP20]], <4 x i32> [[TMP1]])
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP27:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP26]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP27:%.*]] = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i32> [[TMP21]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP28:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP27]])
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -141,7 +127,7 @@ define void @smax_with_invariant_store_user(ptr noalias %src, ptr %dst, i64 %n)
; CHECK-NEXT: [[TMP2]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: store i32 [[TMP4]], ptr [[DST]], align 4
@@ -160,7 +146,7 @@ define void @smax_with_invariant_store_user(ptr noalias %src, ptr %dst, i64 %n)
; CHECK-NEXT: store i32 [[MAX_NEXT]], ptr [[DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -201,7 +187,7 @@ define void @smax_with_multiple_invariant_store_user_same_addr(ptr noalias %src,
; CHECK-NEXT: [[TMP2]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: store i32 [[TMP4]], ptr [[DST]], align 4
@@ -221,7 +207,7 @@ define void @smax_with_multiple_invariant_store_user_same_addr(ptr noalias %src,
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: store i32 [[MAX_NEXT]], ptr [[DST]], align 4
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -301,7 +287,7 @@ define void @smax_with_multiple_invariant_store_user_same_addr3(ptr noalias %src
; CHECK-NEXT: [[TMP2]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: store i32 [[TMP4]], ptr [[DST]], align 4
@@ -321,7 +307,7 @@ define void @smax_with_multiple_invariant_store_user_same_addr3(ptr noalias %src
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: store i32 [[MAX_NEXT]], ptr [[DST]], align 4
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -478,7 +464,7 @@ define i32 @test_predicated_smin(ptr %src) {
; CHECK-NEXT: [[PREDPHI]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP4]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 112
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[PREDPHI]])
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -545,7 +531,7 @@ define i32 @smax_reduction_multiple_incoming(ptr %src, i32 %n, i1 %cond) {
; CHECK-NEXT: [[TMP5]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
@@ -562,7 +548,7 @@ define i32 @smax_reduction_multiple_incoming(ptr %src, i32 %n, i1 %cond) {
; CHECK-NEXT: [[MAX_NEXT]] = tail call i32 @llvm.smax.i32(i32 [[MAX]], i32 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi i32 [ [[MAX_NEXT]], %[[LOOP_HEADER]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[MAX_NEXT_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll
index 97d57a0cf83a0..3b7a0d348fbfb 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll
@@ -26,6 +26,8 @@ define noundef i32 @f(i32 noundef %g) {
; VF4IC2-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
; VF4IC2-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; VF4IC2: [[MIDDLE_BLOCK]]:
+; VF4IC2-NEXT: br label %[[MIDDLE_BLOCK1:.*]]
+; VF4IC2: [[MIDDLE_BLOCK1]]:
; VF4IC2-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
; VF4IC2-NEXT: br label %[[RETURN:.*]]
; VF4IC2: [[VECTOR_EARLY_EXIT]]:
@@ -38,7 +40,7 @@ define noundef i32 @f(i32 noundef %g) {
; VF4IC2-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
; VF4IC2-NEXT: br label %[[RETURN]]
; VF4IC2: [[RETURN]]:
-; VF4IC2-NEXT: [[RES:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT]] ]
+; VF4IC2-NEXT: [[RES:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK1]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT]] ]
; VF4IC2-NEXT: ret i32 [[RES]]
;
; VF8IC1-LABEL: define noundef i32 @f(
@@ -56,6 +58,8 @@ define noundef i32 @f(i32 noundef %g) {
; VF8IC1-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP8]])
; VF8IC1-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; VF8IC1: [[MIDDLE_BLOCK]]:
+; VF8IC1-NEXT: br label %[[MIDDLE_BLOCK1:.*]]
+; VF8IC1: [[MIDDLE_BLOCK1]]:
; VF8IC1-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7
; VF8IC1-NEXT: br label %[[RETURN:.*]]
; VF8IC1: [[VECTOR_EARLY_EXIT]]:
@@ -63,7 +67,7 @@ define noundef i32 @f(i32 noundef %g) {
; VF8IC1-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; VF8IC1-NEXT: br label %[[RETURN]]
; VF8IC1: [[RETURN]]:
-; VF8IC1-NEXT: [[RES:%.*]] = phi i32 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[TMP6]], %[[VECTOR_EARLY_EXIT]] ]
+; VF8IC1-NEXT: [[RES:%.*]] = phi i32 [ [[TMP4]], %[[MIDDLE_BLOCK1]] ], [ [[TMP6]], %[[VECTOR_EARLY_EXIT]] ]
; VF8IC1-NEXT: ret i32 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
index b93215035cebf..1857b5797a5db 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
@@ -53,10 +53,12 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn
; VF8UF2-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; VF8UF2: [[MIDDLE_BLOCK]]:
; VF8UF2-NEXT: br label %[[EXIT:.*]]
-; VF8UF2: [[VECTOR_EARLY_EXIT]]:
-; VF8UF2-NEXT: br label %[[EXIT]]
; VF8UF2: [[EXIT]]:
-; VF8UF2-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[MIDDLE_BLOCK]] ]
+; VF8UF2-NEXT: br label %[[EXIT1:.*]]
+; VF8UF2: [[VECTOR_EARLY_EXIT]]:
+; VF8UF2-NEXT: br label %[[EXIT1]]
+; VF8UF2: [[EXIT1]]:
+; VF8UF2-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[EXIT]] ]
; VF8UF2-NEXT: ret i8 [[RES]]
;
; VF16UF1-LABEL: define i8 @test_early_exit_max_tc_less_than_16(
@@ -73,10 +75,12 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn
; VF16UF1-NEXT: br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; VF16UF1: [[MIDDLE_BLOCK]]:
; VF16UF1-NEXT: br label %[[EXIT:.*]]
-; VF16UF1: [[VECTOR_EARLY_EXIT]]:
-; VF16UF1-NEXT: br label %[[EXIT]]
; VF16UF1: [[EXIT]]:
-; VF16UF1-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[MIDDLE_BLOCK]] ]
+; VF16UF1-NEXT: br label %[[EXIT1:.*]]
+; VF16UF1: [[VECTOR_EARLY_EXIT]]:
+; VF16UF1-NEXT: br label %[[EXIT1]]
+; VF16UF1: [[EXIT1]]:
+; VF16UF1-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[EXIT]] ]
; VF16UF1-NEXT: ret i8 [[RES]]
;
entry:
@@ -147,6 +151,8 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
; VF8UF2-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; VF8UF2: [[MIDDLE_BLOCK]]:
; VF8UF2-NEXT: br label %[[EXIT:.*]]
+; VF8UF2: [[EXIT]]:
+; VF8UF2-NEXT: br label %[[EXIT1:.*]]
; VF8UF2: [[VECTOR_EARLY_EXIT]]:
; VF8UF2-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP2]], i1 false)
; VF8UF2-NEXT: [[TMP7:%.*]] = add i64 8, [[TMP5]]
@@ -154,9 +160,9 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
; VF8UF2-NEXT: [[TMP9:%.*]] = add i64 0, [[TMP8]]
; VF8UF2-NEXT: [[TMP10:%.*]] = icmp ne i64 [[TMP8]], 8
; VF8UF2-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 [[TMP7]]
-; VF8UF2-NEXT: br label %[[EXIT]]
-; VF8UF2: [[EXIT]]:
-; VF8UF2-NEXT: [[RES:%.*]] = phi i64 [ [[TMP11]], %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[MIDDLE_BLOCK]] ]
+; VF8UF2-NEXT: br label %[[EXIT1]]
+; VF8UF2: [[EXIT1]]:
+; VF8UF2-NEXT: [[RES:%.*]] = phi i64 [ [[TMP11]], %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[EXIT]] ]
; VF8UF2-NEXT: ret i64 [[RES]]
;
; VF16UF1-LABEL: define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(
@@ -173,11 +179,13 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
; VF16UF1-NEXT: br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; VF16UF1: [[MIDDLE_BLOCK]]:
; VF16UF1-NEXT: br label %[[EXIT:.*]]
+; VF16UF1: [[EXIT]]:
+; VF16UF1-NEXT: br label %[[EXIT1:.*]]
; VF16UF1: [[VECTOR_EARLY_EXIT]]:
; VF16UF1-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP3]], i1 false)
-; VF16UF1-NEXT: br label %[[EXIT]]
-; VF16UF1: [[EXIT]]:
-; VF16UF1-NEXT: [[RES:%.*]] = phi i64 [ [[FIRST_ACTIVE_LANE]], %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[MIDDLE_BLOCK]] ]
+; VF16UF1-NEXT: br label %[[EXIT1]]
+; VF16UF1: [[EXIT1]]:
+; VF16UF1-NEXT: [[RES:%.*]] = phi i64 [ [[FIRST_ACTIVE_LANE]], %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[EXIT]] ]
; VF16UF1-NEXT: ret i64 [[RES]]
;
entry:
@@ -258,22 +266,24 @@ define i8 @test_early_exit_max_vector_tc_eq_16(ptr dereferenceable(17) %A) nosyn
; VF8UF2-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; VF8UF2: [[MIDDLE_BLOCK]]:
; VF8UF2-NEXT: br label %[[SCALAR_PH:.*]]
-; VF8UF2: [[VECTOR_EARLY_EXIT]]:
-; VF8UF2-NEXT: br label %[[EXIT:.*]]
; VF8UF2: [[SCALAR_PH]]:
+; VF8UF2-NEXT: br label %[[EXIT:.*]]
+; VF8UF2: [[VECTOR_EARLY_EXIT]]:
; VF8UF2-NEXT: br label %[[LOOP_HEADER:.*]]
-; VF8UF2: [[LOOP_HEADER]]:
-; VF8UF2-NEXT: [[IV:%.*]] = phi i64 [ 16, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF8UF2: [[EXIT]]:
+; VF8UF2-NEXT: br label %[[LOOP_HEADER1:.*]]
+; VF8UF2: [[LOOP_HEADER1]]:
+; VF8UF2-NEXT: [[IV:%.*]] = phi i64 [ 16, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; VF8UF2-NEXT: [[P_SRC:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; VF8UF2-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 1
; VF8UF2-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0
-; VF8UF2-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]]
+; VF8UF2-NEXT: br i1 [[C]], label %[[LOOP_HEADER]], label %[[LOOP_LATCH]]
; VF8UF2: [[LOOP_LATCH]]:
; VF8UF2-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; VF8UF2-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 17
-; VF8UF2-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
-; VF8UF2: [[EXIT]]:
-; VF8UF2-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[LOOP_HEADER]] ], [ 1, %[[LOOP_LATCH]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ]
+; VF8UF2-NEXT: br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF8UF2: [[LOOP_HEADER]]:
+; VF8UF2-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[LOOP_HEADER1]] ], [ 1, %[[LOOP_LATCH]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ]
; VF8UF2-NEXT: ret i8 [[RES]]
;
; VF16UF1-LABEL: define i8 @test_early_exit_max_vector_tc_eq_16(
@@ -290,22 +300,24 @@ define i8 @test_early_exit_max_vector_tc_eq_16(ptr dereferenceable(17) %A) nosyn
; VF16UF1-NEXT: br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; VF16UF1: [[MIDDLE_BLOCK]]:
; VF16UF1-NEXT: br label %[[SCALAR_PH:.*]]
-; VF16UF1: [[VECTOR_EARLY_EXIT]]:
-; VF16UF1-NEXT: br label %[[EXIT:.*]]
; VF16UF1: [[SCALAR_PH]]:
+; VF16UF1-NEXT: br label %[[EXIT:.*]]
+; VF16UF1: [[VECTOR_EARLY_EXIT]]:
; VF16UF1-NEXT: br label %[[LOOP_HEADER:.*]]
-; VF16UF1: [[LOOP_HEADER]]:
-; VF16UF1-NEXT: [[IV:%.*]] = phi i64 [ 16, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF16UF1: [[EXIT]]:
+; VF16UF1-NEXT: br label %[[LOOP_HEADER1:.*]]
+; VF16UF1: [[LOOP_HEADER1]]:
+; VF16UF1-NEXT: [[IV:%.*]] = phi i64 [ 16, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; VF16UF1-NEXT: [[P_SRC:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; VF16UF1-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 1
; VF16UF1-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0
-; VF16UF1-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]]
+; VF16UF1-NEXT: br i1 [[C]], label %[[LOOP_HEADER]], label %[[LOOP_LATCH]]
; VF16UF1: [[LOOP_LATCH]]:
; VF16UF1-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; VF16UF1-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 17
-; VF16UF1-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
-; VF16UF1: [[EXIT]]:
-; VF16UF1-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[LOOP_HEADER]] ], [ 1, %[[LOOP_LATCH]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ]
+; VF16UF1-NEXT: br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF16UF1: [[LOOP_HEADER]]:
+; VF16UF1-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[LOOP_HEADER1]] ], [ 1, %[[LOOP_LATCH]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ]
; VF16UF1-NEXT: ret i8 [[RES]]
;
entry:
@@ -368,28 +380,22 @@ define i1 @test_early_exit_max_tc_less_than_16_non_canonical_iv(ptr dereferencea
; VF8UF2: [[VECTOR_PH]]:
; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF8UF2: [[VECTOR_BODY]]:
-; VF8UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; VF8UF2-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
-; VF8UF2-NEXT: [[STEP_ADD:%.*]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
-; VF8UF2-NEXT: [[OFFSET_IDX:%.*]] = add i64 2, [[INDEX]]
-; VF8UF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF8UF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
; VF8UF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
; VF8UF2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
; VF8UF2-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
; VF8UF2-NEXT: [[TMP4:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer
; VF8UF2-NEXT: [[TMP5:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD1]], zeroinitializer
-; VF8UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; VF8UF2-NEXT: [[TMP6:%.*]] = freeze <8 x i1> [[TMP4]]
; VF8UF2-NEXT: [[TMP7:%.*]] = freeze <8 x i1> [[TMP5]]
; VF8UF2-NEXT: [[TMP8:%.*]] = or <8 x i1> [[TMP6]], [[TMP7]]
; VF8UF2-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP8]])
-; VF8UF2-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i64> [[STEP_ADD]], splat (i64 8)
-; VF8UF2-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; VF8UF2-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM:.*]]
; VF8UF2: [[VECTOR_BODY_INTERIM]]:
-; VF8UF2-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF8UF2-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; VF8UF2: [[MIDDLE_BLOCK]]:
; VF8UF2-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_LOAD1]] to <8 x i64>
-; VF8UF2-NEXT: [[TMP3:%.*]] = icmp eq <8 x i64> [[TMP10]], [[STEP_ADD]]
+; VF8UF2-NEXT: [[TMP3:%.*]] = icmp eq <8 x i64> [[TMP10]], <i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17>
; VF8UF2-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[TMP3]], i32 7
; VF8UF2-NEXT: br label %[[EXIT:.*]]
; VF8UF2: [[VECTOR_EARLY_EXIT]]:
@@ -405,22 +411,17 @@ define i1 @test_early_exit_max_tc_less_than_16_non_canonical_iv(ptr dereferencea
; VF16UF1: [[VECTOR_PH]]:
; VF16UF1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF16UF1: [[VECTOR_BODY]]:
-; VF16UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; VF16UF1-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
-; VF16UF1-NEXT: [[OFFSET_IDX:%.*]] = add i64 2, [[INDEX]]
-; VF16UF1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF16UF1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
; VF16UF1-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
; VF16UF1-NEXT: [[TMP3:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD]], zeroinitializer
-; VF16UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; VF16UF1-NEXT: [[TMP4:%.*]] = freeze <16 x i1> [[TMP3]]
; VF16UF1-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP4]])
-; VF16UF1-NEXT: [[VEC_IND_NEXT]] = add nsw <16 x i64> [[VEC_IND]], splat (i64 16)
-; VF16UF1-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; VF16UF1-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM:.*]]
; VF16UF1: [[VECTOR_BODY_INTERIM]]:
-; VF16UF1-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF16UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; VF16UF1: [[MIDDLE_BLOCK]]:
; VF16UF1-NEXT: [[TMP6:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; VF16UF1-NEXT: [[TMP2:%.*]] = icmp eq <16 x i64> [[TMP6]], [[VEC_IND]]
+; VF16UF1-NEXT: [[TMP2:%.*]] = icmp eq <16 x i64> [[TMP6]], <i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17>
; VF16UF1-NEXT: [[TMP8:%.*]] = extractelement <16 x i1> [[TMP2]], i32 15
; VF16UF1-NEXT: br label %[[EXIT:.*]]
; VF16UF1: [[VECTOR_EARLY_EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
index 5da6fc3179043..02846aba50f72 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
@@ -1211,6 +1211,272 @@ loop:
exit:
ret void
}
+
+; Test that a first-order recurrence with a single vector iteration (where the
+; vector loop backedge is removed) does not crash.
+define void @first_order_recurrence_single_vector_iteration(ptr noalias %pkt, ptr noalias %dst) {
+; VF8UF1-LABEL: define void @first_order_recurrence_single_vector_iteration(
+; VF8UF1-SAME: ptr noalias [[PKT:%.*]], ptr noalias [[DST:%.*]]) {
+; VF8UF1-NEXT: [[ENTRY:.*:]]
+; VF8UF1-NEXT: br label %[[VECTOR_PH:.*]]
+; VF8UF1: [[VECTOR_PH]]:
+; VF8UF1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF8UF1: [[VECTOR_BODY]]:
+; VF8UF1-NEXT: [[TMP0:%.*]] = load i8, ptr [[PKT]], align 1
+; VF8UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i64 0
+; VF8UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
+; VF8UF1-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> zeroinitializer, <8 x i8> [[BROADCAST_SPLAT]], <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+; VF8UF1-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; VF8UF1-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
+; VF8UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; VF8UF1: [[MIDDLE_BLOCK]]:
+; VF8UF1-NEXT: br label %[[EXIT:.*]]
+; VF8UF1: [[EXIT]]:
+; VF8UF1-NEXT: ret void
+;
+; VF8UF2-LABEL: define void @first_order_recurrence_single_vector_iteration(
+; VF8UF2-SAME: ptr noalias [[PKT:%.*]], ptr noalias [[DST:%.*]]) {
+; VF8UF2-NEXT: [[ENTRY:.*:]]
+; VF8UF2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF8UF2: [[VECTOR_PH]]:
+; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF8UF2: [[VECTOR_BODY]]:
+; VF8UF2-NEXT: [[TMP0:%.*]] = load i8, ptr [[PKT]], align 1
+; VF8UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i64 0
+; VF8UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
+; VF8UF2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> zeroinitializer, <8 x i8> [[BROADCAST_SPLAT]], <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+; VF8UF2-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLAT]], <8 x i8> [[BROADCAST_SPLAT]], <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF8UF2: [[PRED_STORE_IF]]:
+; VF8UF2-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; VF8UF2-NEXT: store i8 [[TMP3]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VF8UF2: [[PRED_STORE_CONTINUE]]:
+; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF8UF2: [[PRED_STORE_IF1]]:
+; VF8UF2-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; VF8UF2-NEXT: store i8 [[TMP4]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VF8UF2: [[PRED_STORE_CONTINUE2]]:
+; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF8UF2: [[PRED_STORE_IF3]]:
+; VF8UF2-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
+; VF8UF2-NEXT: store i8 [[TMP5]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; VF8UF2: [[PRED_STORE_CONTINUE4]]:
+; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF8UF2: [[PRED_STORE_IF5]]:
+; VF8UF2-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
+; VF8UF2-NEXT: store i8 [[TMP6]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; VF8UF2: [[PRED_STORE_CONTINUE6]]:
+; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; VF8UF2: [[PRED_STORE_IF7]]:
+; VF8UF2-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
+; VF8UF2-NEXT: store i8 [[TMP7]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; VF8UF2: [[PRED_STORE_CONTINUE8]]:
+; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; VF8UF2: [[PRED_STORE_IF9]]:
+; VF8UF2-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
+; VF8UF2-NEXT: store i8 [[TMP8]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; VF8UF2: [[PRED_STORE_CONTINUE10]]:
+; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; VF8UF2: [[PRED_STORE_IF11]]:
+; VF8UF2-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; VF8UF2-NEXT: store i8 [[TMP9]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; VF8UF2: [[PRED_STORE_CONTINUE12]]:
+; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; VF8UF2: [[PRED_STORE_IF13]]:
+; VF8UF2-NEXT: [[TMP10:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; VF8UF2-NEXT: store i8 [[TMP10]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE14]]
+; VF8UF2: [[PRED_STORE_CONTINUE14]]:
+; VF8UF2-NEXT: br i1 false, label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
+; VF8UF2: [[PRED_STORE_IF15]]:
+; VF8UF2-NEXT: [[TMP11:%.*]] = extractelement <8 x i8> [[TMP2]], i32 0
+; VF8UF2-NEXT: store i8 [[TMP11]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; VF8UF2: [[PRED_STORE_CONTINUE16]]:
+; VF8UF2-NEXT: br i1 false, label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
+; VF8UF2: [[PRED_STORE_IF17]]:
+; VF8UF2-NEXT: [[TMP12:%.*]] = extractelement <8 x i8> [[TMP2]], i32 1
+; VF8UF2-NEXT: store i8 [[TMP12]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE18]]
+; VF8UF2: [[PRED_STORE_CONTINUE18]]:
+; VF8UF2-NEXT: br i1 false, label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
+; VF8UF2: [[PRED_STORE_IF19]]:
+; VF8UF2-NEXT: [[TMP13:%.*]] = extractelement <8 x i8> [[TMP2]], i32 2
+; VF8UF2-NEXT: store i8 [[TMP13]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE20]]
+; VF8UF2: [[PRED_STORE_CONTINUE20]]:
+; VF8UF2-NEXT: br i1 false, label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
+; VF8UF2: [[PRED_STORE_IF21]]:
+; VF8UF2-NEXT: [[TMP14:%.*]] = extractelement <8 x i8> [[TMP2]], i32 3
+; VF8UF2-NEXT: store i8 [[TMP14]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE22]]
+; VF8UF2: [[PRED_STORE_CONTINUE22]]:
+; VF8UF2-NEXT: br i1 false, label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
+; VF8UF2: [[PRED_STORE_IF23]]:
+; VF8UF2-NEXT: [[TMP15:%.*]] = extractelement <8 x i8> [[TMP2]], i32 4
+; VF8UF2-NEXT: store i8 [[TMP15]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE24]]
+; VF8UF2: [[PRED_STORE_CONTINUE24]]:
+; VF8UF2-NEXT: br i1 false, label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
+; VF8UF2: [[PRED_STORE_IF25]]:
+; VF8UF2-NEXT: [[TMP16:%.*]] = extractelement <8 x i8> [[TMP2]], i32 5
+; VF8UF2-NEXT: store i8 [[TMP16]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE26]]
+; VF8UF2: [[PRED_STORE_CONTINUE26]]:
+; VF8UF2-NEXT: br i1 false, label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
+; VF8UF2: [[PRED_STORE_IF27]]:
+; VF8UF2-NEXT: [[TMP17:%.*]] = extractelement <8 x i8> [[TMP2]], i32 6
+; VF8UF2-NEXT: store i8 [[TMP17]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE28]]
+; VF8UF2: [[PRED_STORE_CONTINUE28]]:
+; VF8UF2-NEXT: br i1 false, label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
+; VF8UF2: [[PRED_STORE_IF29]]:
+; VF8UF2-NEXT: [[TMP18:%.*]] = extractelement <8 x i8> [[TMP2]], i32 7
+; VF8UF2-NEXT: store i8 [[TMP18]], ptr [[DST]], align 1
+; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE30]]
+; VF8UF2: [[PRED_STORE_CONTINUE30]]:
+; VF8UF2-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; VF8UF2: [[MIDDLE_BLOCK]]:
+; VF8UF2-NEXT: br label %[[EXIT:.*]]
+; VF8UF2: [[EXIT]]:
+; VF8UF2-NEXT: ret void
+;
+; VF16UF1-LABEL: define void @first_order_recurrence_single_vector_iteration(
+; VF16UF1-SAME: ptr noalias [[PKT:%.*]], ptr noalias [[DST:%.*]]) {
+; VF16UF1-NEXT: [[ENTRY:.*:]]
+; VF16UF1-NEXT: br label %[[VECTOR_PH:.*]]
+; VF16UF1: [[VECTOR_PH]]:
+; VF16UF1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF16UF1: [[VECTOR_BODY]]:
+; VF16UF1-NEXT: [[TMP0:%.*]] = load i8, ptr [[PKT]], align 1
+; VF16UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i64 0
+; VF16UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
+; VF16UF1-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> zeroinitializer, <16 x i8> [[BROADCAST_SPLAT]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF16UF1: [[PRED_STORE_IF]]:
+; VF16UF1-NEXT: [[TMP2:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; VF16UF1-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VF16UF1: [[PRED_STORE_CONTINUE]]:
+; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF16UF1: [[PRED_STORE_IF1]]:
+; VF16UF1-NEXT: [[TMP3:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
+; VF16UF1-NEXT: store i8 [[TMP3]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VF16UF1: [[PRED_STORE_CONTINUE2]]:
+; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF16UF1: [[PRED_STORE_IF3]]:
+; VF16UF1-NEXT: [[TMP4:%.*]] = extractelement <16 x i8> [[TMP1]], i32 2
+; VF16UF1-NEXT: store i8 [[TMP4]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; VF16UF1: [[PRED_STORE_CONTINUE4]]:
+; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF16UF1: [[PRED_STORE_IF5]]:
+; VF16UF1-NEXT: [[TMP5:%.*]] = extractelement <16 x i8> [[TMP1]], i32 3
+; VF16UF1-NEXT: store i8 [[TMP5]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; VF16UF1: [[PRED_STORE_CONTINUE6]]:
+; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; VF16UF1: [[PRED_STORE_IF7]]:
+; VF16UF1-NEXT: [[TMP6:%.*]] = extractelement <16 x i8> [[TMP1]], i32 4
+; VF16UF1-NEXT: store i8 [[TMP6]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; VF16UF1: [[PRED_STORE_CONTINUE8]]:
+; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; VF16UF1: [[PRED_STORE_IF9]]:
+; VF16UF1-NEXT: [[TMP7:%.*]] = extractelement <16 x i8> [[TMP1]], i32 5
+; VF16UF1-NEXT: store i8 [[TMP7]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; VF16UF1: [[PRED_STORE_CONTINUE10]]:
+; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; VF16UF1: [[PRED_STORE_IF11]]:
+; VF16UF1-NEXT: [[TMP8:%.*]] = extractelement <16 x i8> [[TMP1]], i32 6
+; VF16UF1-NEXT: store i8 [[TMP8]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; VF16UF1: [[PRED_STORE_CONTINUE12]]:
+; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; VF16UF1: [[PRED_STORE_IF13]]:
+; VF16UF1-NEXT: [[TMP9:%.*]] = extractelement <16 x i8> [[TMP1]], i32 7
+; VF16UF1-NEXT: store i8 [[TMP9]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE14]]
+; VF16UF1: [[PRED_STORE_CONTINUE14]]:
+; VF16UF1-NEXT: br i1 false, label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
+; VF16UF1: [[PRED_STORE_IF15]]:
+; VF16UF1-NEXT: [[TMP10:%.*]] = extractelement <16 x i8> [[TMP1]], i32 8
+; VF16UF1-NEXT: store i8 [[TMP10]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; VF16UF1: [[PRED_STORE_CONTINUE16]]:
+; VF16UF1-NEXT: br i1 false, label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
+; VF16UF1: [[PRED_STORE_IF17]]:
+; VF16UF1-NEXT: [[TMP11:%.*]] = extractelement <16 x i8> [[TMP1]], i32 9
+; VF16UF1-NEXT: store i8 [[TMP11]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE18]]
+; VF16UF1: [[PRED_STORE_CONTINUE18]]:
+; VF16UF1-NEXT: br i1 false, label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
+; VF16UF1: [[PRED_STORE_IF19]]:
+; VF16UF1-NEXT: [[TMP12:%.*]] = extractelement <16 x i8> [[TMP1]], i32 10
+; VF16UF1-NEXT: store i8 [[TMP12]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE20]]
+; VF16UF1: [[PRED_STORE_CONTINUE20]]:
+; VF16UF1-NEXT: br i1 false, label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
+; VF16UF1: [[PRED_STORE_IF21]]:
+; VF16UF1-NEXT: [[TMP13:%.*]] = extractelement <16 x i8> [[TMP1]], i32 11
+; VF16UF1-NEXT: store i8 [[TMP13]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE22]]
+; VF16UF1: [[PRED_STORE_CONTINUE22]]:
+; VF16UF1-NEXT: br i1 false, label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
+; VF16UF1: [[PRED_STORE_IF23]]:
+; VF16UF1-NEXT: [[TMP14:%.*]] = extractelement <16 x i8> [[TMP1]], i32 12
+; VF16UF1-NEXT: store i8 [[TMP14]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE24]]
+; VF16UF1: [[PRED_STORE_CONTINUE24]]:
+; VF16UF1-NEXT: br i1 false, label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
+; VF16UF1: [[PRED_STORE_IF25]]:
+; VF16UF1-NEXT: [[TMP15:%.*]] = extractelement <16 x i8> [[TMP1]], i32 13
+; VF16UF1-NEXT: store i8 [[TMP15]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE26]]
+; VF16UF1: [[PRED_STORE_CONTINUE26]]:
+; VF16UF1-NEXT: br i1 false, label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
+; VF16UF1: [[PRED_STORE_IF27]]:
+; VF16UF1-NEXT: [[TMP16:%.*]] = extractelement <16 x i8> [[TMP1]], i32 14
+; VF16UF1-NEXT: store i8 [[TMP16]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE28]]
+; VF16UF1: [[PRED_STORE_CONTINUE28]]:
+; VF16UF1-NEXT: br i1 false, label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
+; VF16UF1: [[PRED_STORE_IF29]]:
+; VF16UF1-NEXT: [[TMP17:%.*]] = extractelement <16 x i8> [[TMP1]], i32 15
+; VF16UF1-NEXT: store i8 [[TMP17]], ptr [[DST]], align 1
+; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE30]]
+; VF16UF1: [[PRED_STORE_CONTINUE30]]:
+; VF16UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; VF16UF1: [[MIDDLE_BLOCK]]:
+; VF16UF1-NEXT: br label %[[EXIT:.*]]
+; VF16UF1: [[EXIT]]:
+; VF16UF1-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %recur = phi i8 [ 0, %entry ], [ %load, %loop ]
+ %load = load i8, ptr %pkt, align 1
+ store i8 %recur, ptr %dst, align 1
+ %iv.next = add i64 %iv, 1
+ %cmp = icmp eq i64 %iv, 7
+ br i1 %cmp, label %exit, label %loop
+
+exit:
+ ret void
+}
+;. !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
;.
; VF8UF1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; VF8UF1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
>From da8d1b52d0d344068a8bd3e46a8af3a692d2efc2 Mon Sep 17 00:00:00 2001
From: Yue Huang <30948580+AdUhTkJm at users.noreply.github.com>
Date: Sun, 8 Mar 2026 11:21:18 +0000
Subject: [PATCH 18/25] [MLIR][Presburger] Fix simplify() of IntegerRelation
(#181469)
In `simplify()`, we currently skip all columns from 0 to `firstVar` (the
column of the pivot) when we eliminate inequalities. This is invalid,
because unlike equalities, it is not guaranteed that these columns
contain only zeroes, and a scale by `rowMultiplier` cannot be ignored.
We must not skip any columns for inequalities.
---
mlir/lib/Analysis/Presburger/IntegerRelation.cpp | 2 +-
.../Analysis/Presburger/IntegerRelationTest.cpp | 10 ++++++++++
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
index cf93fbd9a0dc7..93a725571078e 100644
--- a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
+++ b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
@@ -1162,7 +1162,7 @@ bool IntegerRelation::gaussianEliminate() {
equalities.normalizeRow(i);
}
for (unsigned i = 0, ineqs = getNumInequalities(); i < ineqs; ++i) {
- eliminateFromConstraint(this, i, *pivotRow, firstVar, 0, false);
+ eliminateFromConstraint(this, i, *pivotRow, firstVar, firstVar, false);
inequalities.normalizeRow(i);
}
gcdTightenInequalities();
diff --git a/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp b/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
index b94b86057b650..90753d502d12c 100644
--- a/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
+++ b/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
@@ -741,6 +741,16 @@ TEST(IntegerRelationTest, simplify) {
EXPECT_TRUE(rel.getNumEqualities() == 2);
}
+TEST(IntegerRelationTest, simplifyRegression) {
+ IntegerRelation rel = parseRelationFromSet("(x, y, z): (2*x + z >= 5, "
+ "x + 3*z >= 7, 3*x + y >= 9)",
+ 3);
+
+ auto simplified = rel;
+ simplified.simplify();
+ EXPECT_TRUE(rel.isEqual(simplified));
+}
+
TEST(IntegerRelationTest, isFullDim) {
IntegerRelation rel = parseRelationFromSet("(x): (1 >= 0)", 1);
EXPECT_TRUE(rel.isFullDim());
>From d8b718a3fa540827a7c1bf250fd55cfa0d029e0f Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 8 Mar 2026 05:31:49 -0700
Subject: [PATCH 19/25] [SLP]Match the mask size, when copying mask for full
match
Need to be careful, when filling the mask for fully matched nodes, the
masks may differ in sizes
Fixes a crash reported in test/Transforms/SLPVectorizer/X86/mask-size-less-common-mask.ll
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
.../X86/mask-size-less-common-mask.ll | 187 ++++++++++++++++++
2 files changed, 188 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/mask-size-less-common-mask.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 8bc16972beddc..591ee2fea3148 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18496,7 +18496,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
TE->ReuseShuffleIndices.size() == VL.size() &&
(*It)->isSame(TE->Scalars)))) {
Entries.push_back(*It);
- if ((*It)->getVectorFactor() == VL.size()) {
+ if (IsReusedNodeFound || (*It)->getVectorFactor() == VL.size()) {
std::iota(std::next(Mask.begin(), Part * VL.size()),
std::next(Mask.begin(), (Part + 1) * VL.size()), 0);
} else {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/mask-size-less-common-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/mask-size-less-common-mask.ll
new file mode 100644
index 0000000000000..3db2925ad1b50
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/mask-size-less-common-mask.ll
@@ -0,0 +1,187 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[CMP8_8_I:%.*]] = icmp ult i8 [[TMP0]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i8 [[TMP0]], 0
+; CHECK-NEXT: [[NARROW_8_I:%.*]] = select i1 [[CMP8_8_I]], i8 0, i8 [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX13_8_I:%.*]] = getelementptr i8, ptr null, i64 16
+; CHECK-NEXT: store i8 [[NARROW_8_I]], ptr [[ARRAYIDX13_8_I]], align 1
+; CHECK-NEXT: [[CMP15_8_I:%.*]] = icmp ult i8 [[TMP0]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = and i8 [[TMP0]], 0
+; CHECK-NEXT: [[NARROW31_8_I:%.*]] = select i1 [[CMP15_8_I]], i8 0, i8 [[TMP2]]
+; CHECK-NEXT: [[ARRAYIDX25_8_I:%.*]] = getelementptr i8, ptr null, i64 17
+; CHECK-NEXT: store i8 [[NARROW31_8_I]], ptr [[ARRAYIDX25_8_I]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[CMP8_9_I:%.*]] = icmp ult i8 [[TMP3]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i8 [[TMP3]], 0
+; CHECK-NEXT: [[NARROW_9_I:%.*]] = select i1 [[CMP8_9_I]], i8 0, i8 [[TMP4]]
+; CHECK-NEXT: [[ARRAYIDX13_9_I:%.*]] = getelementptr i8, ptr null, i64 18
+; CHECK-NEXT: store i8 [[NARROW_9_I]], ptr [[ARRAYIDX13_9_I]], align 1
+; CHECK-NEXT: [[CMP15_9_I:%.*]] = icmp ult i8 [[TMP3]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = and i8 [[TMP3]], 0
+; CHECK-NEXT: [[NARROW31_9_I:%.*]] = select i1 [[CMP15_9_I]], i8 0, i8 [[TMP5]]
+; CHECK-NEXT: [[ARRAYIDX25_9_I:%.*]] = getelementptr i8, ptr null, i64 19
+; CHECK-NEXT: store i8 [[NARROW31_9_I]], ptr [[ARRAYIDX25_9_I]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[CMP8_10_I:%.*]] = icmp ult i8 [[TMP6]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = lshr i8 [[TMP6]], 0
+; CHECK-NEXT: [[NARROW_10_I:%.*]] = select i1 [[CMP8_10_I]], i8 0, i8 [[TMP7]]
+; CHECK-NEXT: [[ARRAYIDX13_10_I:%.*]] = getelementptr i8, ptr null, i64 20
+; CHECK-NEXT: store i8 [[NARROW_10_I]], ptr [[ARRAYIDX13_10_I]], align 1
+; CHECK-NEXT: [[CMP15_10_I:%.*]] = icmp ult i8 [[TMP6]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = and i8 [[TMP6]], 0
+; CHECK-NEXT: [[NARROW31_10_I:%.*]] = select i1 [[CMP15_10_I]], i8 0, i8 [[TMP8]]
+; CHECK-NEXT: [[ARRAYIDX25_10_I:%.*]] = getelementptr i8, ptr null, i64 21
+; CHECK-NEXT: store i8 [[NARROW31_10_I]], ptr [[ARRAYIDX25_10_I]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[CMP8_11_I:%.*]] = icmp ult i8 [[TMP9]], 0
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i8 [[TMP9]], 0
+; CHECK-NEXT: [[NARROW_11_I:%.*]] = select i1 [[CMP8_11_I]], i8 0, i8 [[TMP10]]
+; CHECK-NEXT: [[ARRAYIDX13_11_I:%.*]] = getelementptr i8, ptr null, i64 22
+; CHECK-NEXT: store i8 [[NARROW_11_I]], ptr [[ARRAYIDX13_11_I]], align 1
+; CHECK-NEXT: [[CMP15_11_I:%.*]] = icmp ult i8 [[TMP9]], 0
+; CHECK-NEXT: [[TMP11:%.*]] = and i8 [[TMP9]], 0
+; CHECK-NEXT: [[NARROW31_11_I:%.*]] = select i1 [[CMP15_11_I]], i8 0, i8 [[TMP11]]
+; CHECK-NEXT: [[ARRAYIDX25_11_I:%.*]] = getelementptr i8, ptr null, i64 23
+; CHECK-NEXT: store i8 [[NARROW31_11_I]], ptr [[ARRAYIDX25_11_I]], align 1
+; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[CMP8_12_I:%.*]] = icmp ult i8 [[TMP12]], 0
+; CHECK-NEXT: [[TMP13:%.*]] = lshr i8 [[TMP12]], 0
+; CHECK-NEXT: [[NARROW_12_I:%.*]] = select i1 [[CMP8_12_I]], i8 0, i8 [[TMP13]]
+; CHECK-NEXT: [[ARRAYIDX13_12_I:%.*]] = getelementptr i8, ptr null, i64 24
+; CHECK-NEXT: store i8 [[NARROW_12_I]], ptr [[ARRAYIDX13_12_I]], align 1
+; CHECK-NEXT: [[CMP15_12_I:%.*]] = icmp ult i8 [[TMP12]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = and i8 [[TMP12]], 0
+; CHECK-NEXT: [[NARROW31_12_I:%.*]] = select i1 [[CMP15_12_I]], i8 0, i8 [[TMP14]]
+; CHECK-NEXT: [[ARRAYIDX25_12_I:%.*]] = getelementptr i8, ptr null, i64 25
+; CHECK-NEXT: store i8 [[NARROW31_12_I]], ptr [[ARRAYIDX25_12_I]], align 1
+; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[CMP8_13_I:%.*]] = icmp ult i8 [[TMP15]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = lshr i8 [[TMP15]], 0
+; CHECK-NEXT: [[NARROW_13_I:%.*]] = select i1 [[CMP8_13_I]], i8 0, i8 [[TMP16]]
+; CHECK-NEXT: [[ARRAYIDX13_13_I:%.*]] = getelementptr i8, ptr null, i64 26
+; CHECK-NEXT: store i8 [[NARROW_13_I]], ptr [[ARRAYIDX13_13_I]], align 1
+; CHECK-NEXT: [[CMP15_13_I:%.*]] = icmp ult i8 [[TMP15]], 0
+; CHECK-NEXT: [[TMP17:%.*]] = and i8 [[TMP15]], 0
+; CHECK-NEXT: [[NARROW31_13_I:%.*]] = select i1 [[CMP15_13_I]], i8 0, i8 [[TMP17]]
+; CHECK-NEXT: [[ARRAYIDX25_13_I:%.*]] = getelementptr i8, ptr null, i64 27
+; CHECK-NEXT: store i8 [[NARROW31_13_I]], ptr [[ARRAYIDX25_13_I]], align 1
+; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[CMP8_14_I:%.*]] = icmp ult i8 [[TMP18]], 0
+; CHECK-NEXT: [[TMP19:%.*]] = lshr i8 [[TMP18]], 0
+; CHECK-NEXT: [[NARROW_14_I:%.*]] = select i1 [[CMP8_14_I]], i8 0, i8 [[TMP19]]
+; CHECK-NEXT: [[ARRAYIDX13_14_I:%.*]] = getelementptr i8, ptr null, i64 28
+; CHECK-NEXT: store i8 [[NARROW_14_I]], ptr [[ARRAYIDX13_14_I]], align 1
+; CHECK-NEXT: [[CMP15_14_I:%.*]] = icmp ult i8 [[TMP18]], 0
+; CHECK-NEXT: [[TMP20:%.*]] = and i8 [[TMP18]], 0
+; CHECK-NEXT: [[NARROW31_14_I:%.*]] = select i1 [[CMP15_14_I]], i8 0, i8 [[TMP20]]
+; CHECK-NEXT: [[ARRAYIDX25_14_I:%.*]] = getelementptr i8, ptr null, i64 29
+; CHECK-NEXT: store i8 [[NARROW31_14_I]], ptr [[ARRAYIDX25_14_I]], align 1
+; CHECK-NEXT: [[TMP21:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[CMP8_15_I:%.*]] = icmp ult i8 [[TMP21]], 0
+; CHECK-NEXT: [[TMP22:%.*]] = lshr i8 [[TMP21]], 0
+; CHECK-NEXT: [[NARROW_15_I:%.*]] = select i1 [[CMP8_15_I]], i8 0, i8 [[TMP22]]
+; CHECK-NEXT: [[ARRAYIDX13_15_I:%.*]] = getelementptr i8, ptr null, i64 30
+; CHECK-NEXT: store i8 [[NARROW_15_I]], ptr [[ARRAYIDX13_15_I]], align 1
+; CHECK-NEXT: [[CMP15_15_I:%.*]] = icmp ult i8 [[TMP21]], 0
+; CHECK-NEXT: [[TMP23:%.*]] = and i8 [[TMP21]], 0
+; CHECK-NEXT: [[NARROW31_15_I:%.*]] = select i1 [[CMP15_15_I]], i8 0, i8 [[TMP23]]
+; CHECK-NEXT: [[ARRAYIDX25_15_I:%.*]] = getelementptr i8, ptr null, i64 31
+; CHECK-NEXT: store i8 [[NARROW31_15_I]], ptr [[ARRAYIDX25_15_I]], align 1
+; CHECK-NEXT: ret void
+;
+entry:
+ %0 = load i8, ptr null, align 1
+ %cmp8.8.i = icmp ult i8 %0, 0
+ %1 = lshr i8 %0, 0
+ %narrow.8.i = select i1 %cmp8.8.i, i8 0, i8 %1
+ %arrayidx13.8.i = getelementptr i8, ptr null, i64 16
+ store i8 %narrow.8.i, ptr %arrayidx13.8.i, align 1
+ %cmp15.8.i = icmp ult i8 %0, 0
+ %2 = and i8 %0, 0
+ %narrow31.8.i = select i1 %cmp15.8.i, i8 0, i8 %2
+ %arrayidx25.8.i = getelementptr i8, ptr null, i64 17
+ store i8 %narrow31.8.i, ptr %arrayidx25.8.i, align 1
+ %3 = load i8, ptr null, align 1
+ %cmp8.9.i = icmp ult i8 %3, 0
+ %4 = lshr i8 %3, 0
+ %narrow.9.i = select i1 %cmp8.9.i, i8 0, i8 %4
+ %arrayidx13.9.i = getelementptr i8, ptr null, i64 18
+ store i8 %narrow.9.i, ptr %arrayidx13.9.i, align 1
+ %cmp15.9.i = icmp ult i8 %3, 0
+ %5 = and i8 %3, 0
+ %narrow31.9.i = select i1 %cmp15.9.i, i8 0, i8 %5
+ %arrayidx25.9.i = getelementptr i8, ptr null, i64 19
+ store i8 %narrow31.9.i, ptr %arrayidx25.9.i, align 1
+ %6 = load i8, ptr null, align 1
+ %cmp8.10.i = icmp ult i8 %6, 0
+ %7 = lshr i8 %6, 0
+ %narrow.10.i = select i1 %cmp8.10.i, i8 0, i8 %7
+ %arrayidx13.10.i = getelementptr i8, ptr null, i64 20
+ store i8 %narrow.10.i, ptr %arrayidx13.10.i, align 1
+ %cmp15.10.i = icmp ult i8 %6, 0
+ %8 = and i8 %6, 0
+ %narrow31.10.i = select i1 %cmp15.10.i, i8 0, i8 %8
+ %arrayidx25.10.i = getelementptr i8, ptr null, i64 21
+ store i8 %narrow31.10.i, ptr %arrayidx25.10.i, align 1
+ %9 = load i8, ptr null, align 1
+ %cmp8.11.i = icmp ult i8 %9, 0
+ %10 = lshr i8 %9, 0
+ %narrow.11.i = select i1 %cmp8.11.i, i8 0, i8 %10
+ %arrayidx13.11.i = getelementptr i8, ptr null, i64 22
+ store i8 %narrow.11.i, ptr %arrayidx13.11.i, align 1
+ %cmp15.11.i = icmp ult i8 %9, 0
+ %11 = and i8 %9, 0
+ %narrow31.11.i = select i1 %cmp15.11.i, i8 0, i8 %11
+ %arrayidx25.11.i = getelementptr i8, ptr null, i64 23
+ store i8 %narrow31.11.i, ptr %arrayidx25.11.i, align 1
+ %12 = load i8, ptr null, align 1
+ %cmp8.12.i = icmp ult i8 %12, 0
+ %13 = lshr i8 %12, 0
+ %narrow.12.i = select i1 %cmp8.12.i, i8 0, i8 %13
+ %arrayidx13.12.i = getelementptr i8, ptr null, i64 24
+ store i8 %narrow.12.i, ptr %arrayidx13.12.i, align 1
+ %cmp15.12.i = icmp ult i8 %12, 0
+ %14 = and i8 %12, 0
+ %narrow31.12.i = select i1 %cmp15.12.i, i8 0, i8 %14
+ %arrayidx25.12.i = getelementptr i8, ptr null, i64 25
+ store i8 %narrow31.12.i, ptr %arrayidx25.12.i, align 1
+ %15 = load i8, ptr null, align 1
+ %cmp8.13.i = icmp ult i8 %15, 0
+ %16 = lshr i8 %15, 0
+ %narrow.13.i = select i1 %cmp8.13.i, i8 0, i8 %16
+ %arrayidx13.13.i = getelementptr i8, ptr null, i64 26
+ store i8 %narrow.13.i, ptr %arrayidx13.13.i, align 1
+ %cmp15.13.i = icmp ult i8 %15, 0
+ %17 = and i8 %15, 0
+ %narrow31.13.i = select i1 %cmp15.13.i, i8 0, i8 %17
+ %arrayidx25.13.i = getelementptr i8, ptr null, i64 27
+ store i8 %narrow31.13.i, ptr %arrayidx25.13.i, align 1
+ %18 = load i8, ptr null, align 1
+ %cmp8.14.i = icmp ult i8 %18, 0
+ %19 = lshr i8 %18, 0
+ %narrow.14.i = select i1 %cmp8.14.i, i8 0, i8 %19
+ %arrayidx13.14.i = getelementptr i8, ptr null, i64 28
+ store i8 %narrow.14.i, ptr %arrayidx13.14.i, align 1
+ %cmp15.14.i = icmp ult i8 %18, 0
+ %20 = and i8 %18, 0
+ %narrow31.14.i = select i1 %cmp15.14.i, i8 0, i8 %20
+ %arrayidx25.14.i = getelementptr i8, ptr null, i64 29
+ store i8 %narrow31.14.i, ptr %arrayidx25.14.i, align 1
+ %21 = load i8, ptr null, align 1
+ %cmp8.15.i = icmp ult i8 %21, 0
+ %22 = lshr i8 %21, 0
+ %narrow.15.i = select i1 %cmp8.15.i, i8 0, i8 %22
+ %arrayidx13.15.i = getelementptr i8, ptr null, i64 30
+ store i8 %narrow.15.i, ptr %arrayidx13.15.i, align 1
+ %cmp15.15.i = icmp ult i8 %21, 0
+ %23 = and i8 %21, 0
+ %narrow31.15.i = select i1 %cmp15.15.i, i8 0, i8 %23
+ %arrayidx25.15.i = getelementptr i8, ptr null, i64 31
+ store i8 %narrow31.15.i, ptr %arrayidx25.15.i, align 1
+ ret void
+}
>From 2bf2115471278a214212d589d429e25bcf7f1241 Mon Sep 17 00:00:00 2001
From: Arjun Parmar <akparmar0404 at gmail.com>
Date: Sun, 8 Mar 2026 18:13:16 +0530
Subject: [PATCH 20/25] [DAG] isKnownToBeAPowerOfTwo - add ISD::VECTOR_SHUFFLE
handling (#185203)
closes #183350
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 24 ++++++++++++++
llvm/test/CodeGen/X86/known-pow2.ll | 31 +++++++++++++++++++
2 files changed, 55 insertions(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index a2318fc034fa2..4ec771d7fd41f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4784,6 +4784,30 @@ bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
// vscale(power-of-two) is a power-of-two
return isKnownToBeAPowerOfTwo(Val.getOperand(0), /*OrZero=*/false,
Depth + 1);
+
+ case ISD::VECTOR_SHUFFLE: {
+ assert(!Val.getValueType().isScalableVector());
+ // Demanded elements with undef shuffle mask elements are unknown
+ // - we cannot guarantee they are a power of two, so return false.
+ APInt DemandedLHS, DemandedRHS;
+ const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
+ assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
+ if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
+ DemandedLHS, DemandedRHS))
+ return false;
+
+ // All demanded elements from LHS must be known power of two.
+ if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedLHS,
+ OrZero, Depth + 1))
+ return false;
+
+ // All demanded elements from RHS must be known power of two.
+ if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val.getOperand(1), DemandedRHS,
+ OrZero, Depth + 1))
+ return false;
+
+ return true;
+ }
}
// More could be done here, though the above checks are enough
diff --git a/llvm/test/CodeGen/X86/known-pow2.ll b/llvm/test/CodeGen/X86/known-pow2.ll
index a9b28458e9518..fa852e4c9ec2f 100644
--- a/llvm/test/CodeGen/X86/known-pow2.ll
+++ b/llvm/test/CodeGen/X86/known-pow2.ll
@@ -1241,3 +1241,34 @@ define i32 @pow2_rotr_extract_vec(<4 x i32> %a0, <4 x i32> %rotamt, i32 %x, ptr
%res = urem i32 %x, %elt
ret i32 %res
}
+
+define <4 x i32> @pow2_shuffle_vec(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
+; CHECK-LABEL: pow2_shuffle_vec:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pxor %xmm3, %xmm3
+; CHECK-NEXT: pxor %xmm4, %xmm4
+; CHECK-NEXT: pcmpgtd %xmm0, %xmm4
+; CHECK-NEXT: pcmpgtd %xmm1, %xmm3
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [8,4,2,4294967295]
+; CHECK-NEXT: movdqa %xmm4, %xmm1
+; CHECK-NEXT: pandn %xmm0, %xmm1
+; CHECK-NEXT: movdqa {{.*#+}} xmm5 = [4,2,4294967295,0]
+; CHECK-NEXT: pand %xmm5, %xmm4
+; CHECK-NEXT: por %xmm1, %xmm4
+; CHECK-NEXT: movdqa %xmm3, %xmm1
+; CHECK-NEXT: pandn %xmm5, %xmm1
+; CHECK-NEXT: pand %xmm0, %xmm3
+; CHECK-NEXT: por %xmm1, %xmm3
+; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; CHECK-NEXT: pcmpeqd %xmm0, %xmm0
+; CHECK-NEXT: paddd %xmm4, %xmm0
+; CHECK-NEXT: pand %xmm2, %xmm0
+; CHECK-NEXT: retq
+ %cmp0 = icmp sgt <4 x i32> zeroinitializer, %a0
+ %cmp1 = icmp sgt <4 x i32> zeroinitializer, %a1
+ %sel0 = select <4 x i1> %cmp0, <4 x i32> <i32 4, i32 2, i32 -1, i32 0>, <4 x i32> <i32 8, i32 4, i32 2, i32 -1>
+ %sel1 = select <4 x i1> %cmp1, <4 x i32> <i32 8, i32 4, i32 2, i32 -1>, <4 x i32> <i32 4, i32 2, i32 -1, i32 0>
+ %shuf = shufflevector <4 x i32> %sel0, <4 x i32> %sel1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+ %res = urem <4 x i32> %a2, %shuf
+ ret <4 x i32> %res
+}
>From 82ef7e54d3e644a26a834b240f590287cf3ee352 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Sun, 8 Mar 2026 13:10:01 +0000
Subject: [PATCH 21/25] [X86] combineVTRUNCSAT - attempt to recognise
VTRUNCS/US(CONCAT(X,Y)) -> PACKSS/US(X,Y) folds. (#178707)
If we're just concatenating subvectors together to perform a saturated
truncate, see if we can perform PACK on the subvectors directly instead
- 256-bit PACK will require a post-shuffle, but this will typically fold
away in later shuffle combining and its probably better than changing
vector widths with concats.
Reference patch based off poor codegen identified in #169995
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 37 +++++++++++++++
llvm/test/CodeGen/X86/masked_packss.ll | 20 ++-------
llvm/test/CodeGen/X86/masked_packus.ll | 28 ++----------
llvm/test/CodeGen/X86/packss.ll | 52 +++++----------------
llvm/test/CodeGen/X86/packus.ll | 60 +++++--------------------
5 files changed, 65 insertions(+), 132 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1ebfd5defdc40..8959ca924d2af 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -55898,6 +55898,41 @@ static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+static SDValue combineVTRUNCSAT(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ using namespace SDPatternMatch;
+ unsigned Opc = N->getOpcode();
+ EVT VT = N->getValueType(0);
+ unsigned EltSizeInBits = VT.getScalarSizeInBits();
+ assert((Opc == X86ISD::VTRUNCS || Opc == X86ISD::VTRUNCUS) &&
+ "Unexpected VTRUNC node");
+ assert((VT.is128BitVector() || VT.is256BitVector()) &&
+ "Unexpected VTRUNC node type");
+
+ // If the src was concatenated, see if PACKSS/US would be better.
+ SDValue Src;
+ if (EltSizeInBits <= 16 &&
+ (sd_match(N, m_UnaryOp(X86ISD::VTRUNCS, m_Value(Src))) ||
+ sd_match(N, m_UnaryOp(X86ISD::VTRUNCUS,
+ m_SMaxLike(m_Value(Src), m_Zero())))) &&
+ (EltSizeInBits * 2) == Src.getScalarValueSizeInBits() &&
+ isFreeToSplitVector(Src, DAG)) {
+ SDLoc DL(N);
+ auto [LHS, RHS] = splitVector(Src, DAG, DL);
+ unsigned PackOpc = Opc == X86ISD::VTRUNCS ? X86ISD::PACKSS : X86ISD::PACKUS;
+ SDValue Pack = DAG.getNode(PackOpc, DL, VT, LHS, RHS);
+ if (VT.is128BitVector())
+ return Pack;
+ // Shuffle sub-lanes back to match the vtrunc sequential order.
+ return DAG.getBitcast(
+ VT, DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64,
+ DAG.getBitcast(MVT::v4i64, Pack),
+ getV4X86ShuffleImm8ForMask({0, 2, 1, 3}, DL, DAG)));
+ }
+
+ return SDValue();
+}
+
/// Returns the negated value if the node \p N flips sign of FP value.
///
/// FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000)
@@ -62506,6 +62541,8 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget);
case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget);
case X86ISD::VTRUNC: return combineVTRUNC(N, DAG, DCI);
+ case X86ISD::VTRUNCS:
+ case X86ISD::VTRUNCUS: return combineVTRUNCSAT(N, DAG, DCI);
case X86ISD::ANDNP: return combineAndnp(N, DAG, DCI, Subtarget);
case X86ISD::FAND: return combineFAnd(N, DAG, Subtarget);
case X86ISD::FANDN: return combineFAndn(N, DAG, Subtarget);
diff --git a/llvm/test/CodeGen/X86/masked_packss.ll b/llvm/test/CodeGen/X86/masked_packss.ll
index 9daf74539776c..d84eaeaae60b7 100644
--- a/llvm/test/CodeGen/X86/masked_packss.ll
+++ b/llvm/test/CodeGen/X86/masked_packss.ll
@@ -16,11 +16,8 @@ define <16 x i8> @_mm_mask_packss_epi16_manual(<16 x i8> %src, i16 noundef %k, <
;
; AVX512-LABEL: _mm_mask_packss_epi16_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512-NEXT: kmovd %edi, %k1
-; AVX512-NEXT: vpmovswb %ymm1, %xmm0 {%k1}
-; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: vpacksswb %xmm2, %xmm1, %xmm0 {%k1}
; AVX512-NEXT: retq
%sh = shufflevector <8 x i16> %a, <8 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%minv = tail call <16 x i16> @llvm.smax.v16i16(<16 x i16> %sh, <16 x i16> splat (i16 -128))
@@ -46,11 +43,8 @@ define <32 x i8> @_mm256_mask_packss_epi16_manual(<32 x i8> %src, i32 noundef %k
;
; AVX512-LABEL: _mm256_mask_packss_epi16_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],ymm2[2,3]
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
; AVX512-NEXT: kmovd %edi, %k1
-; AVX512-NEXT: vpmovswb %zmm1, %ymm0 {%k1}
+; AVX512-NEXT: vpacksswb %ymm2, %ymm1, %ymm0 {%k1}
; AVX512-NEXT: retq
%sh = shufflevector <16 x i16> %a, <16 x i16> %b, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
%minv = tail call <32 x i16> @llvm.smax.v32i16(<32 x i16> %sh, <32 x i16> splat (i16 -128))
@@ -114,11 +108,8 @@ define <8 x i16> @_mm_mask_packss_epi32_manual(<8 x i16> %src, i8 noundef %k, <4
;
; AVX512-LABEL: _mm_mask_packss_epi32_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512-NEXT: kmovd %edi, %k1
-; AVX512-NEXT: vpmovsdw %ymm1, %xmm0 {%k1}
-; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: vpackssdw %xmm2, %xmm1, %xmm0 {%k1}
; AVX512-NEXT: retq
%sh = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%minv = tail call <8 x i32> @llvm.smax.v8i32(<8 x i32> %sh, <8 x i32> splat (i32 -32768))
@@ -143,11 +134,8 @@ define <16 x i16> @_mm256_mask_packss_epi32_manual(<16 x i16> %src, i16 noundef
;
; AVX512-LABEL: _mm256_mask_packss_epi32_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],ymm2[2,3]
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
; AVX512-NEXT: kmovd %edi, %k1
-; AVX512-NEXT: vpmovsdw %zmm1, %ymm0 {%k1}
+; AVX512-NEXT: vpackssdw %ymm2, %ymm1, %ymm0 {%k1}
; AVX512-NEXT: retq
%sh = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
%minv = tail call <16 x i32> @llvm.smax.v16i32(<16 x i32> %sh, <16 x i32> splat (i32 -32768))
diff --git a/llvm/test/CodeGen/X86/masked_packus.ll b/llvm/test/CodeGen/X86/masked_packus.ll
index 65aa90173e3e6..52507542945c7 100644
--- a/llvm/test/CodeGen/X86/masked_packus.ll
+++ b/llvm/test/CodeGen/X86/masked_packus.ll
@@ -16,13 +16,8 @@ define <16 x i8> @_mm_mask_packus_epi16_manual(<16 x i8> %src, i16 noundef %k, <
;
; AVX512-LABEL: _mm_mask_packus_epi16_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpmaxsw %ymm2, %ymm1, %ymm1
; AVX512-NEXT: kmovd %edi, %k1
-; AVX512-NEXT: vpmovuswb %ymm1, %xmm0 {%k1}
-; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: vpackuswb %xmm2, %xmm1, %xmm0 {%k1}
; AVX512-NEXT: retq
%sh = shufflevector <8 x i16> %a, <8 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%minv = tail call <16 x i16> @llvm.smax.v16i16(<16 x i16> %sh, <16 x i16> splat (i16 0))
@@ -48,13 +43,8 @@ define <32 x i8> @_mm256_mask_packus_epi16_manual(<32 x i8> %src, i32 noundef %k
;
; AVX512-LABEL: _mm256_mask_packus_epi16_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],ymm2[2,3]
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpmaxsw %zmm2, %zmm1, %zmm1
; AVX512-NEXT: kmovd %edi, %k1
-; AVX512-NEXT: vpmovuswb %zmm1, %ymm0 {%k1}
+; AVX512-NEXT: vpackuswb %ymm2, %ymm1, %ymm0 {%k1}
; AVX512-NEXT: retq
%sh = shufflevector <16 x i16> %a, <16 x i16> %b, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
%minv = tail call <32 x i16> @llvm.smax.v32i16(<32 x i16> %sh, <32 x i16> splat (i16 0))
@@ -121,13 +111,8 @@ define <8 x i16> @_mm_mask_packus_epi32_manual(<8 x i16> %src, i8 noundef %k, <4
;
; AVX512-LABEL: _mm_mask_packus_epi32_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpmaxsd %ymm2, %ymm1, %ymm1
; AVX512-NEXT: kmovd %edi, %k1
-; AVX512-NEXT: vpmovusdw %ymm1, %xmm0 {%k1}
-; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: vpackusdw %xmm2, %xmm1, %xmm0 {%k1}
; AVX512-NEXT: retq
%sh = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%minv = tail call <8 x i32> @llvm.smax.v8i32(<8 x i32> %sh, <8 x i32> splat (i32 0))
@@ -152,13 +137,8 @@ define <16 x i16> @_mm256_mask_packus_epi32_manual(<16 x i16> %src, i16 noundef
;
; AVX512-LABEL: _mm256_mask_packus_epi32_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],ymm2[2,3]
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpmaxsd %zmm2, %zmm1, %zmm1
; AVX512-NEXT: kmovd %edi, %k1
-; AVX512-NEXT: vpmovusdw %zmm1, %ymm0 {%k1}
+; AVX512-NEXT: vpackusdw %ymm2, %ymm1, %ymm0 {%k1}
; AVX512-NEXT: retq
%sh = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
%minv = tail call <16 x i32> @llvm.smax.v16i32(<16 x i32> %sh, <16 x i32> splat (i32 0))
diff --git a/llvm/test/CodeGen/X86/packss.ll b/llvm/test/CodeGen/X86/packss.ll
index 1c61b1eb84cd5..da739dc277f68 100644
--- a/llvm/test/CodeGen/X86/packss.ll
+++ b/llvm/test/CodeGen/X86/packss.ll
@@ -431,23 +431,10 @@ define <16 x i8> @_mm_packss_epi16_manual(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE-NEXT: packsswb %xmm1, %xmm0
; SSE-NEXT: ret{{[l|q]}}
;
-; AVX1-LABEL: _mm_packss_epi16_manual:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpacksswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: ret{{[l|q]}}
-;
-; AVX2-LABEL: _mm_packss_epi16_manual:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpacksswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: ret{{[l|q]}}
-;
-; AVX512-LABEL: _mm_packss_epi16_manual:
-; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpmovswb %ymm0, %xmm0
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: ret{{[l|q]}}
+; AVX-LABEL: _mm_packss_epi16_manual:
+; AVX: # %bb.0:
+; AVX-NEXT: vpacksswb %xmm1, %xmm0, %xmm0
+; AVX-NEXT: ret{{[l|q]}}
%sh = shufflevector <8 x i16> %a, <8 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%minv = tail call <16 x i16> @llvm.smax.v16i16(<16 x i16> %sh, <16 x i16> splat (i16 -128))
%sat = tail call <16 x i16> @llvm.smin.v16i16(<16 x i16> %minv, <16 x i16> splat (i16 127))
@@ -484,10 +471,7 @@ define <32 x i8> @_mm256_packss_epi16_manual(<16 x i16> %a, <16 x i16> %b) nounw
;
; AVX512-LABEL: _mm256_packss_epi16_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT: vpmovswb %zmm0, %ymm0
+; AVX512-NEXT: vpacksswb %ymm1, %ymm0, %ymm0
; AVX512-NEXT: ret{{[l|q]}}
;
; X64-SSE-LABEL: _mm256_packss_epi16_manual:
@@ -600,23 +584,10 @@ define <8 x i16> @_mm_packss_epi32_manual(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE-NEXT: packssdw %xmm1, %xmm0
; SSE-NEXT: ret{{[l|q]}}
;
-; AVX1-LABEL: _mm_packss_epi32_manual:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: ret{{[l|q]}}
-;
-; AVX2-LABEL: _mm_packss_epi32_manual:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: ret{{[l|q]}}
-;
-; AVX512-LABEL: _mm_packss_epi32_manual:
-; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpmovsdw %ymm0, %xmm0
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: ret{{[l|q]}}
+; AVX-LABEL: _mm_packss_epi32_manual:
+; AVX: # %bb.0:
+; AVX-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: ret{{[l|q]}}
%sh = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%minv = tail call <8 x i32> @llvm.smax.v8i32(<8 x i32> %sh, <8 x i32> splat (i32 -32768))
%sat = tail call <8 x i32> @llvm.smin.v8i32(<8 x i32> %minv, <8 x i32> splat (i32 32767))
@@ -653,10 +624,7 @@ define <16 x i16> @_mm256_packss_epi32_manual(<8 x i32> %a, <8 x i32> %b) nounwi
;
; AVX512-LABEL: _mm256_packss_epi32_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512-NEXT: vpackssdw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: ret{{[l|q]}}
;
; X64-SSE-LABEL: _mm256_packss_epi32_manual:
diff --git a/llvm/test/CodeGen/X86/packus.ll b/llvm/test/CodeGen/X86/packus.ll
index c0941ef19d5c0..38fd914f1f947 100644
--- a/llvm/test/CodeGen/X86/packus.ll
+++ b/llvm/test/CodeGen/X86/packus.ll
@@ -517,25 +517,10 @@ define <16 x i8> @_mm_packus_epi16_manual(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE-NEXT: packuswb %xmm1, %xmm0
; SSE-NEXT: ret{{[l|q]}}
;
-; AVX1-LABEL: _mm_packus_epi16_manual:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: ret{{[l|q]}}
-;
-; AVX2-LABEL: _mm_packus_epi16_manual:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: ret{{[l|q]}}
-;
-; AVX512-LABEL: _mm_packus_epi16_manual:
-; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpmovuswb %ymm0, %xmm0
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: ret{{[l|q]}}
+; AVX-LABEL: _mm_packus_epi16_manual:
+; AVX: # %bb.0:
+; AVX-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX-NEXT: ret{{[l|q]}}
%sh = shufflevector <8 x i16> %a, <8 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%minv = tail call <16 x i16> @llvm.smax.v16i16(<16 x i16> %sh, <16 x i16> splat (i16 0))
%sat = tail call <16 x i16> @llvm.smin.v16i16(<16 x i16> %minv, <16 x i16> splat (i16 255))
@@ -578,12 +563,7 @@ define <32 x i8> @_mm256_packus_epi16_manual(<16 x i16> %a, <16 x i16> %b) nounw
;
; AVX512-LABEL: _mm256_packus_epi16_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpmovuswb %zmm0, %ymm0
+; AVX512-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
; AVX512-NEXT: ret{{[l|q]}}
%sh = shufflevector <16 x i16> %a, <16 x i16> %b, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
%minv = tail call <32 x i16> @llvm.smax.v32i16(<32 x i16> %sh, <32 x i16> splat (i16 0))
@@ -747,25 +727,10 @@ define <8 x i16> @_mm_packus_epi32_manual(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE4-NEXT: packusdw %xmm1, %xmm0
; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX1-LABEL: _mm_packus_epi32_manual:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: ret{{[l|q]}}
-;
-; AVX2-LABEL: _mm_packus_epi32_manual:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: ret{{[l|q]}}
-;
-; AVX512-LABEL: _mm_packus_epi32_manual:
-; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpmovusdw %ymm0, %xmm0
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: ret{{[l|q]}}
+; AVX-LABEL: _mm_packus_epi32_manual:
+; AVX: # %bb.0:
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: ret{{[l|q]}}
%sh = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%minv = tail call <8 x i32> @llvm.smax.v32i32(<8 x i32> %sh, <8 x i32> splat (i32 0))
%sat = tail call <8 x i32> @llvm.smin.v32i32(<8 x i32> %minv, <8 x i32> splat (i32 65535))
@@ -911,12 +876,7 @@ define <16 x i16> @_mm256_packus_epi32_manual(<8 x i32> %a, <8 x i32> %b) nounwi
;
; AVX512-LABEL: _mm256_packus_epi32_manual:
; AVX512: # %bb.0:
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpmovusdw %zmm0, %ymm0
+; AVX512-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: ret{{[l|q]}}
%sh = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
%minv = tail call <16 x i32> @llvm.smax.v32i32(<16 x i32> %sh, <16 x i32> splat (i32 0))
>From 421bc73cffe783fd82903d3d335ac238376b4682 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <185856+superbobry at users.noreply.github.com>
Date: Sun, 8 Mar 2026 13:22:04 +0000
Subject: [PATCH 22/25] [MLIR] [Python] Fixed a few issues in the type
annotations (#183021)
* Removed an explicit `nb::sig` for `static_typeid`. The inferred type
would
work just fine, and unqualified `TypeID`, which was there previously,
only
really works for core types in the `ir` submodule.
* `DefaultingPyMlir*` helpers also produce qualified types, e.g.
`_mlir.ir.Location` instead of bare `Location`.
* `ir.*.__enter__` now returns a concrete type instead of `object`, e.g.
`ir.Context.__enter__` returns `Context`.
* `loc_tracebacks` uses `Generator` as the return type, since this is
what
`contextmanager` expects in typeshed.
* Changed static methods on subclasses of `DenseElementsAttribute` to
return
that concrete subclass, instead of `DenseElementsAttribute`.
---------
Co-authored-by: Maksim Levental <maksim.levental at gmail.com>
---
mlir/cmake/modules/AddMLIRPython.cmake | 29 ++++++++-
.../mlir/Bindings/Python/IRAttributes.h | 7 ++
mlir/include/mlir/Bindings/Python/IRCore.h | 36 ++++-------
mlir/lib/Bindings/Python/IRAttributes.cpp | 64 ++++++++++++-------
mlir/lib/Bindings/Python/IRCore.cpp | 12 ++--
mlir/python/mlir/ir.py | 4 +-
6 files changed, 101 insertions(+), 51 deletions(-)
diff --git a/mlir/cmake/modules/AddMLIRPython.cmake b/mlir/cmake/modules/AddMLIRPython.cmake
index 1821cfbf35d2a..6ac5003538e45 100644
--- a/mlir/cmake/modules/AddMLIRPython.cmake
+++ b/mlir/cmake/modules/AddMLIRPython.cmake
@@ -123,8 +123,35 @@ function(mlir_generate_type_stubs)
"IMPORT_PATHS;DEPENDS_TARGETS;OUTPUTS;DEPENDS_TARGET_SRC_DEPS"
${ARGN})
+ # Allow overriding the stubgen.py path or fetching a specific version
+ # from the nanobind repository, independent of the nanobind used for
+ # building. This is useful when a newer stubgen has bug fixes or features
+ # not yet available in the nanobind version used for compilation.
+ if(MLIR_NB_STUBGEN)
+ set(NB_STUBGEN "${MLIR_NB_STUBGEN}")
+ elseif(MLIR_NB_STUBGEN_VERSION)
+ set(_stubgen_path "${MLIR_BINARY_DIR}/stubgen/${MLIR_NB_STUBGEN_VERSION}/stubgen.py")
+ if(NOT EXISTS "${_stubgen_path}")
+ message(STATUS "Downloading stubgen.py from nanobind ${MLIR_NB_STUBGEN_VERSION}...")
+ file(MAKE_DIRECTORY "${MLIR_BINARY_DIR}/stubgen/${MLIR_NB_STUBGEN_VERSION}" RESULT _created_dir)
+ if(NOT _created_dir EQUAL 0)
+ list(GET _created_dir 1 _created_dir_error)
+ message(FATAL_ERROR "Failed to create parent dir for stubgen.py: ${_created_dir_error}")
+ endif()
+ file(DOWNLOAD
+ "https://raw.githubusercontent.com/wjakob/nanobind/${MLIR_NB_STUBGEN_VERSION}/src/stubgen.py"
+ "${_stubgen_path}"
+ STATUS _download_status
+ )
+ list(GET _download_status 0 _download_code)
+ if(NOT _download_code EQUAL 0)
+ list(GET _download_status 1 _download_error)
+ message(FATAL_ERROR "Failed to download stubgen.py: ${_download_error}")
+ endif()
+ endif()
+ set(NB_STUBGEN "${_stubgen_path}")
# for people installing a distro (e.g., pip install) of nanobind
- if(EXISTS ${nanobind_DIR}/../src/stubgen.py)
+ elseif(EXISTS ${nanobind_DIR}/../src/stubgen.py)
set(NB_STUBGEN "${nanobind_DIR}/../src/stubgen.py")
elseif(EXISTS ${nanobind_DIR}/../stubgen.py)
set(NB_STUBGEN "${nanobind_DIR}/../stubgen.py")
diff --git a/mlir/include/mlir/Bindings/Python/IRAttributes.h b/mlir/include/mlir/Bindings/Python/IRAttributes.h
index 173674e0091d2..64a31d3f3e42d 100644
--- a/mlir/include/mlir/Bindings/Python/IRAttributes.h
+++ b/mlir/include/mlir/Bindings/Python/IRAttributes.h
@@ -424,6 +424,13 @@ class MLIR_PYTHON_API_EXPORTED PyDenseElementsAttribute
static PyType_Slot slots[];
+protected:
+ /// Registers get/get_splat factory methods with the concrete return
+ /// type in the nb::sig. Subclasses call this from their bindDerived
+ /// to override the return type in generated stubs.
+ template <typename ClassT>
+ static void bindFactoryMethods(ClassT &c, const char *pyClassName);
+
private:
static int bf_getbuffer(PyObject *exporter, Py_buffer *view, int flags);
static void bf_releasebuffer(PyObject *, Py_buffer *buffer);
diff --git a/mlir/include/mlir/Bindings/Python/IRCore.h b/mlir/include/mlir/Bindings/Python/IRCore.h
index 5953f26d07370..bd2d49acbf681 100644
--- a/mlir/include/mlir/Bindings/Python/IRCore.h
+++ b/mlir/include/mlir/Bindings/Python/IRCore.h
@@ -278,7 +278,7 @@ class MLIR_PYTHON_API_EXPORTED DefaultingPyMlirContext
: public Defaulting<DefaultingPyMlirContext, PyMlirContext> {
public:
using Defaulting::Defaulting;
- static constexpr const char kTypeDescription[] = "Context";
+ static constexpr const char kTypeDescription[] = "_mlir.ir.Context";
static PyMlirContext &resolve();
};
@@ -524,7 +524,7 @@ class MLIR_PYTHON_API_EXPORTED DefaultingPyLocation
: public Defaulting<DefaultingPyLocation, PyLocation> {
public:
using Defaulting::Defaulting;
- static constexpr const char kTypeDescription[] = "Location";
+ static constexpr const char kTypeDescription[] = "_mlir.ir.Location";
static PyLocation &resolve();
operator MlirLocation() const { return *get(); }
@@ -957,16 +957,12 @@ class MLIR_PYTHON_API_EXPORTED PyConcreteType : public BaseTy {
auto cls = ClassTy(m, DerivedTy::pyClassName, nanobind::is_generic());
cls.def(nanobind::init<PyType &>(), nanobind::keep_alive<0, 1>(),
nanobind::arg("cast_from_type"));
- cls.def_prop_ro_static(
- "static_typeid",
- [](nanobind::object & /*class*/) {
- if (DerivedTy::getTypeIdFunction)
- return PyTypeID(DerivedTy::getTypeIdFunction());
- throw nanobind::attribute_error(
- (DerivedTy::pyClassName + std::string(" has no typeid."))
- .c_str());
- },
- nanobind::sig("def static_typeid(/) -> TypeID"));
+ cls.def_prop_ro_static("static_typeid", [](nanobind::object & /*class*/) {
+ if (DerivedTy::getTypeIdFunction)
+ return PyTypeID(DerivedTy::getTypeIdFunction());
+ throw nanobind::attribute_error(
+ (DerivedTy::pyClassName + std::string(" has no typeid.")).c_str());
+ });
cls.def_prop_ro("typeid", [](PyType &self) {
return nanobind::cast<PyTypeID>(nanobind::cast(self).attr("typeid"));
});
@@ -1100,16 +1096,12 @@ class MLIR_PYTHON_API_EXPORTED PyConcreteAttribute : public BaseTy {
return PyType(attr.getContext(), mlirAttributeGetType(attr))
.maybeDownCast();
});
- cls.def_prop_ro_static(
- "static_typeid",
- [](nanobind::object & /*class*/) -> PyTypeID {
- if (DerivedTy::getTypeIdFunction)
- return PyTypeID(DerivedTy::getTypeIdFunction());
- throw nanobind::attribute_error(
- (DerivedTy::pyClassName + std::string(" has no typeid."))
- .c_str());
- },
- nanobind::sig("def static_typeid(/) -> TypeID"));
+ cls.def_prop_ro_static("static_typeid", [](nanobind::object & /*class*/) {
+ if (DerivedTy::getTypeIdFunction)
+ return PyTypeID(DerivedTy::getTypeIdFunction());
+ throw nanobind::attribute_error(
+ (DerivedTy::pyClassName + std::string(" has no typeid.")).c_str());
+ });
cls.def_prop_ro("typeid", [](PyAttribute &self) {
return nanobind::cast<PyTypeID>(nanobind::cast(self).attr("typeid"));
});
diff --git a/mlir/lib/Bindings/Python/IRAttributes.cpp b/mlir/lib/Bindings/Python/IRAttributes.cpp
index 59eb9b8e81cf0..997f16978fa58 100644
--- a/mlir/lib/Bindings/Python/IRAttributes.cpp
+++ b/mlir/lib/Bindings/Python/IRAttributes.cpp
@@ -421,12 +421,9 @@ void PyIntegerAttribute::bindDerived(ClassTy &c) {
c.def_prop_ro("value", toPyInt, "Returns the value of the integer attribute");
c.def("__int__", toPyInt,
"Converts the value of the integer attribute to a Python int");
- c.def_prop_ro_static(
- "static_typeid",
- [](nb::object & /*class*/) {
- return PyTypeID(mlirIntegerAttrGetTypeID());
- },
- nb::sig("def static_typeid(/) -> TypeID"));
+ c.def_prop_ro_static("static_typeid", [](nb::object & /*class*/) {
+ return PyTypeID(mlirIntegerAttrGetTypeID());
+ });
}
nb::object PyIntegerAttribute::toPyInt(PyIntegerAttribute &self) {
@@ -774,27 +771,48 @@ std::unique_ptr<nb_buffer_info> PyDenseElementsAttribute::accessBuffer() {
"unsupported data type for conversion to Python buffer");
}
-void PyDenseElementsAttribute::bindDerived(ClassTy &c) {
- c.def("__len__", &PyDenseElementsAttribute::dunderLen)
- .def_static(
- "get", PyDenseElementsAttribute::getFromBuffer, nb::arg("array"),
- nb::arg("signless") = true, nb::arg("type") = nb::none(),
- nb::arg("shape") = nb::none(), nb::arg("context") = nb::none(),
- // clang-format off
- nb::sig("def get(array: typing_extensions.Buffer, signless: bool = True, type: Type | None = None, shape: Sequence[int] | None = None, context: Context | None = None) -> DenseElementsAttr"),
- // clang-format on
- kDenseElementsAttrGetDocstring)
+template <typename ClassT>
+void PyDenseElementsAttribute::bindFactoryMethods(ClassT &c,
+ const char *pyClassName) {
+ std::string getSig1 =
+ // clang-format off
+ "def get(array: typing_extensions.Buffer, signless: bool = True, type: Type | None = None, shape: Sequence[int] | None = None, context: Context | None = None) -> " +
+ // clang-format on
+ std::string(pyClassName);
+ std::string getSig2 =
+ // clang-format off
+ "def get(attrs: Sequence[Attribute], type: Type | None = None, context: Context | None = None) -> " +
+ // clang-format on
+ std::string(pyClassName);
+ std::string getSplatSig =
+ // clang-format off
+ "def get_splat(shaped_type: Type, element_attr: Attribute) -> " +
+ // clang-format on
+ std::string(pyClassName);
+
+ c.def_static("get", PyDenseElementsAttribute::getFromBuffer, nb::arg("array"),
+ nb::arg("signless") = true, nb::arg("type") = nb::none(),
+ nb::arg("shape") = nb::none(), nb::arg("context") = nb::none(),
+ nb::sig(getSig1.c_str()), kDenseElementsAttrGetDocstring)
.def_static("get", PyDenseElementsAttribute::getFromList,
nb::arg("attrs"), nb::arg("type") = nb::none(),
- nb::arg("context") = nb::none(),
+ nb::arg("context") = nb::none(), nb::sig(getSig2.c_str()),
kDenseElementsAttrGetFromListDocstring)
.def_static("get_splat", PyDenseElementsAttribute::getSplat,
nb::arg("shaped_type"), nb::arg("element_attr"),
- "Gets a DenseElementsAttr where all values are the same")
- .def_prop_ro("is_splat",
- [](PyDenseElementsAttribute &self) -> bool {
- return mlirDenseElementsAttrIsSplat(self);
- })
+ nb::sig(getSplatSig.c_str()),
+ ("Gets a " + std::string(pyClassName) +
+ " where all values are the same")
+ .c_str());
+}
+
+void PyDenseElementsAttribute::bindDerived(ClassTy &c) {
+ c.def("__len__", &PyDenseElementsAttribute::dunderLen);
+ bindFactoryMethods(c, pyClassName);
+ c.def_prop_ro("is_splat",
+ [](PyDenseElementsAttribute &self) -> bool {
+ return mlirDenseElementsAttrIsSplat(self);
+ })
.def("get_splat_value",
[](PyDenseElementsAttribute &self)
-> nb::typed<nb::object, PyAttribute> {
@@ -1037,6 +1055,7 @@ nb::int_ PyDenseIntElementsAttribute::dunderGetItem(intptr_t pos) const {
}
void PyDenseIntElementsAttribute::bindDerived(ClassTy &c) {
+ PyDenseElementsAttribute::bindFactoryMethods(c, pyClassName);
c.def("__getitem__", &PyDenseIntElementsAttribute::dunderGetItem);
}
@@ -1215,6 +1234,7 @@ nb::float_ PyDenseFPElementsAttribute::dunderGetItem(intptr_t pos) const {
}
void PyDenseFPElementsAttribute::bindDerived(ClassTy &c) {
+ PyDenseElementsAttribute::bindFactoryMethods(c, pyClassName);
c.def("__getitem__", &PyDenseFPElementsAttribute::dunderGetItem);
}
diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
index 4d389c656e58d..b8637c57a3f48 100644
--- a/mlir/lib/Bindings/Python/IRCore.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -2988,7 +2988,8 @@ void populateIRCore(nb::module_ &m) {
"Returns True if an error was encountered during diagnostic "
"handling.")
.def("__enter__", &PyDiagnosticHandler::contextEnter,
- "Enters the diagnostic handler as a context manager.")
+ "Enters the diagnostic handler as a context manager.",
+ nb::sig("def __enter__(self, /) -> DiagnosticHandler"))
.def("__exit__", &PyDiagnosticHandler::contextExit, "exc_type"_a.none(),
"exc_value"_a.none(), "traceback"_a.none(),
"Exits the diagnostic handler context manager.");
@@ -3034,7 +3035,8 @@ void populateIRCore(nb::module_ &m) {
&PyMlirContext::createFromCapsule,
"Creates a Context from a capsule wrapping MlirContext.")
.def("__enter__", &PyMlirContext::contextEnter,
- "Enters the context as a context manager.")
+ "Enters the context as a context manager.",
+ nb::sig("def __enter__(self, /) -> Context"))
.def("__exit__", &PyMlirContext::contextExit, "exc_type"_a.none(),
"exc_value"_a.none(), "traceback"_a.none(),
"Exits the context manager.")
@@ -3260,7 +3262,8 @@ void populateIRCore(nb::module_ &m) {
.def_static(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyLocation::createFromCapsule,
"Creates a Location from a capsule wrapping MlirLocation.")
.def("__enter__", &PyLocation::contextEnter,
- "Enters the location as a context manager.")
+ "Enters the location as a context manager.",
+ nb::sig("def __enter__(self, /) -> Location"))
.def("__exit__", &PyLocation::contextExit, "exc_type"_a.none(),
"exc_value"_a.none(), "traceback"_a.none(),
"Exits the location context manager.")
@@ -4290,7 +4293,8 @@ void populateIRCore(nb::module_ &m) {
.def(nb::init<PyBlock &>(), "block"_a,
"Inserts after the last operation but still inside the block.")
.def("__enter__", &PyInsertionPoint::contextEnter,
- "Enters the insertion point as a context manager.")
+ "Enters the insertion point as a context manager.",
+ nb::sig("def __enter__(self, /) -> InsertionPoint"))
.def("__exit__", &PyInsertionPoint::contextExit, "exc_type"_a.none(),
"exc_value"_a.none(), "traceback"_a.none(),
"Exits the insertion point context manager.")
diff --git a/mlir/python/mlir/ir.py b/mlir/python/mlir/ir.py
index f4aa2d6b051c8..f84792d4095f4 100644
--- a/mlir/python/mlir/ir.py
+++ b/mlir/python/mlir/ir.py
@@ -4,7 +4,7 @@
from __future__ import annotations
-from collections.abc import Iterable
+from collections.abc import Generator
from contextlib import contextmanager
from ._mlir_libs._mlir.ir import *
@@ -22,7 +22,7 @@
@contextmanager
-def loc_tracebacks(*, max_depth: int | None = None) -> Iterable[None]:
+def loc_tracebacks(*, max_depth: int | None = None) -> Generator[None]:
"""Enables automatic traceback-based locations for MLIR operations.
Operations created within this context will have their location
>From e0e5000ea70b115a69ca8802003ac9be56907d4f Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 8 Mar 2026 09:37:51 -0400
Subject: [PATCH 23/25] [SLP]Remove Alternate early profitability checks in
favor of throttling
Removes early check, which may prevent some further optimizations, in
favor of tree throttling.
Reviewers: RKSimon, hiraditya
Pull Request: https://github.com/llvm/llvm-project/pull/182760
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 140 +-----------------
.../AArch64/externally-used-copyables.ll | 12 +-
.../RISCV/shuffled-gather-casted.ll | 16 +-
.../X86/gather-move-out-of-loop.ll | 26 +---
...gathered-delayed-nodes-with-reused-user.ll | 21 ---
.../non-scheduled-inst-reused-as-last-inst.ll | 19 ---
.../X86/reorder_with_external_users.ll | 4 +-
.../X86/shl-to-add-transformation5.ll | 13 +-
.../SLPVectorizer/X86/slp-throttle.ll | 8 +-
.../small-graph-diff-block-instructions.ll | 9 +-
.../SLPVectorizer/alternate-non-profitable.ll | 51 ++++---
.../ext-int-reduced-not-operand.ll | 30 +---
12 files changed, 85 insertions(+), 264 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 591ee2fea3148..2e33a6a5d3303 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -129,11 +129,6 @@ static cl::opt<int>
cl::desc("Only vectorize if you gain more than this "
"number "));
-static cl::opt<bool> SLPSkipEarlyProfitabilityCheck(
- "slp-skip-early-profitability-check", cl::init(false), cl::Hidden,
- cl::desc("When true, SLP vectorizer bypasses profitability checks based on "
- "heuristics and makes vectorization decision via cost modeling."));
-
static cl::opt<bool>
ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
cl::desc("Attempt to vectorize horizontal reductions"));
@@ -4608,15 +4603,6 @@ class slpvectorizer::BoUpSLP {
return nullptr;
}
- /// Check that the operand node of alternate node does not generate
- /// buildvector sequence. If it is, then probably not worth it to build
- /// alternate shuffle, if number of buildvector operands + alternate
- /// instruction > than the number of buildvector instructions.
- /// \param S the instructions state of the analyzed values.
- /// \param VL list of the instructions with alternate opcodes.
- bool areAltOperandsProfitable(const InstructionsState &S,
- ArrayRef<Value *> VL) const;
-
/// Contains all the outputs of legality analysis for a list of values to
/// vectorize.
class ScalarsVectorizationLegality {
@@ -10244,120 +10230,6 @@ static std::pair<size_t, size_t> generateKeySubkey(
static bool isMainInstruction(Instruction *I, Instruction *MainOp,
Instruction *AltOp, const TargetLibraryInfo &TLI);
-bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
- ArrayRef<Value *> VL) const {
- Type *ScalarTy = S.getMainOp()->getType();
- unsigned Opcode0 = S.getOpcode();
- unsigned Opcode1 = S.getAltOpcode();
- SmallBitVector OpcodeMask(getAltInstrMask(VL, ScalarTy, Opcode0, Opcode1));
- // If this pattern is supported by the target then consider it profitable.
- if (TTI->isLegalAltInstr(getWidenedType(ScalarTy, VL.size()), Opcode0,
- Opcode1, OpcodeMask))
- return true;
- SmallVector<ValueList> Operands;
- for (unsigned I : seq<unsigned>(S.getMainOp()->getNumOperands())) {
- Operands.emplace_back();
- // Prepare the operand vector.
- for (Value *V : VL) {
- if (isa<PoisonValue>(V)) {
- Operands.back().push_back(
- PoisonValue::get(S.getMainOp()->getOperand(I)->getType()));
- continue;
- }
- Operands.back().push_back(cast<Instruction>(V)->getOperand(I));
- }
- }
- if (Operands.size() == 2) {
- // Try find best operands candidates.
- for (unsigned I : seq<unsigned>(0, VL.size() - 1)) {
- SmallVector<std::pair<Value *, Value *>> Candidates(3);
- Candidates[0] = std::make_pair(Operands[0][I], Operands[0][I + 1]);
- Candidates[1] = std::make_pair(Operands[0][I], Operands[1][I + 1]);
- Candidates[2] = std::make_pair(Operands[1][I], Operands[0][I + 1]);
- std::optional<int> Res = findBestRootPair(Candidates);
- switch (Res.value_or(0)) {
- case 0:
- break;
- case 1:
- std::swap(Operands[0][I + 1], Operands[1][I + 1]);
- break;
- case 2:
- std::swap(Operands[0][I], Operands[1][I]);
- break;
- default:
- llvm_unreachable("Unexpected index.");
- }
- }
- }
- DenseSet<unsigned> UniqueOpcodes;
- constexpr unsigned NumAltInsts = 3; // main + alt + shuffle.
- unsigned NonInstCnt = 0;
- // Estimate number of instructions, required for the vectorized node and for
- // the buildvector node.
- unsigned UndefCnt = 0;
- // Count the number of extra shuffles, required for vector nodes.
- unsigned ExtraShuffleInsts = 0;
- // Check that operands do not contain same values and create either perfect
- // diamond match or shuffled match.
- if (Operands.size() == 2) {
- // Do not count same operands twice.
- if (Operands.front() == Operands.back()) {
- Operands.erase(Operands.begin());
- } else if (!allConstant(Operands.front()) &&
- all_of(Operands.front(), [&](Value *V) {
- return is_contained(Operands.back(), V);
- })) {
- Operands.erase(Operands.begin());
- ++ExtraShuffleInsts;
- }
- }
- const Loop *L = LI->getLoopFor(S.getMainOp()->getParent());
- // Vectorize node, if:
- // 1. at least single operand is constant or splat.
- // 2. Operands have many loop invariants (the instructions are not loop
- // invariants).
- // 3. At least single unique operands is supposed to vectorized.
- return none_of(Operands,
- [&](ArrayRef<Value *> Op) {
- if (allConstant(Op) ||
- (!isSplat(Op) && allSameBlock(Op) && allSameType(Op) &&
- getSameOpcode(Op, *TLI)))
- return false;
- DenseMap<Value *, unsigned> Uniques;
- for (Value *V : Op) {
- if (isa<Constant, ExtractElementInst>(V) ||
- isVectorized(V) || (L && L->isLoopInvariant(V))) {
- if (isa<UndefValue>(V))
- ++UndefCnt;
- continue;
- }
- auto Res = Uniques.try_emplace(V, 0);
- // Found first duplicate - need to add shuffle.
- if (!Res.second && Res.first->second == 1)
- ++ExtraShuffleInsts;
- ++Res.first->getSecond();
- if (auto *I = dyn_cast<Instruction>(V))
- UniqueOpcodes.insert(I->getOpcode());
- else if (Res.second)
- ++NonInstCnt;
- }
- return none_of(Uniques, [&](const auto &P) {
- return P.first->hasNUsesOrMore(P.second + 1) &&
- none_of(P.first->users(), [&](User *U) {
- return isVectorized(U) || Uniques.contains(U);
- });
- });
- }) ||
- // Do not vectorize node, if estimated number of vector instructions is
- // more than estimated number of buildvector instructions. Number of
- // vector operands is number of vector instructions + number of vector
- // instructions for operands (buildvectors). Number of buildvector
- // instructions is just number_of_operands * number_of_scalars.
- (UndefCnt < (VL.size() - 1) * S.getMainOp()->getNumOperands() &&
- (UniqueOpcodes.size() + NonInstCnt + ExtraShuffleInsts +
- NumAltInsts) < S.getMainOp()->getNumOperands() * VL.size());
-}
-
/// Builds the arguments types vector for the given call instruction with the
/// given \p ID for the specified vector factor.
static SmallVector<Type *>
@@ -10827,13 +10699,6 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
return TreeEntry::NeedToGather;
}
- if (!SLPSkipEarlyProfitabilityCheck && !areAltOperandsProfitable(S, VL)) {
- LLVM_DEBUG(
- dbgs()
- << "SLP: ShuffleVector not vectorized, operands are buildvector and "
- "the whole alt sequence is not profitable.\n");
- return TreeEntry::NeedToGather;
- }
return TreeEntry::Vectorize;
}
@@ -17274,6 +17139,8 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
auto It = MinBWs.find(TE);
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
+ if (isa<CmpInst>(TE->Scalars.front()))
+ ScalarTy = TE->Scalars.front()->getType();
auto *VecTy = getWidenedType(ScalarTy, Sz);
const unsigned EntryVF = TE->getVectorFactor();
auto *FinalVecTy = getWidenedType(ScalarTy, EntryVF);
@@ -17302,7 +17169,8 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, FinalVecTy, Mask);
// If all scalars are reused in gather node(s) or other vector nodes, there
// might be extra cost for inserting them.
- if (all_of(TE->Scalars, [&](Value *V) {
+ if ((!TE->hasState() || !TE->isAltShuffle()) &&
+ all_of(TE->Scalars, [&](Value *V) {
return (TE->hasCopyableElements() && TE->isCopyableElement(V)) ||
isConstant(V) || isGathered(V) || getTreeEntries(V).size() > 1;
}))
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
index 38705032ce1c9..77a1c812c52a0 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
@@ -34,6 +34,13 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-NEXT: [[TMP80:%.*]] = insertelement <64 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 1, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison>, i64 [[TMP1]], i32 11
; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> poison, <28 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP84:%.*]] = shufflevector <4 x i64> [[TMP17]], <4 x i64> poison, <14 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <2 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <2 x i64> [[TMP45]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <2 x i64> [[TMP85]], <2 x i64> <i64 poison, i64 1>, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP87:%.*]] = mul <2 x i64> [[TMP85]], [[TMP86]]
+; CHECK-NEXT: [[TMP88:%.*]] = or <2 x i64> [[TMP85]], [[TMP86]]
+; CHECK-NEXT: [[TMP89:%.*]] = shufflevector <2 x i64> [[TMP87]], <2 x i64> [[TMP88]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP90:%.*]] = shufflevector <2 x i64> [[TMP89]], <2 x i64> poison, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: br label %[[DOTLR_PH1977_US:.*]]
; CHECK: [[_LR_PH1977_US:.*:]]
; CHECK-NEXT: [[INDVAR37888:%.*]] = phi i64 [ 0, [[DOTLR_PH_PREHEADER:%.*]] ], [ 1, %[[DOTLR_PH1977_US]] ]
@@ -41,15 +48,12 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <4 x i64> [[TMP34]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 3, i32 3>
; CHECK-NEXT: [[TMP36:%.*]] = mul <4 x i64> [[TMP20]], [[TMP31]]
; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP27:%.*]] = mul i64 [[TMP0]], [[TMP0]]
-; CHECK-NEXT: [[TMP30:%.*]] = or i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP40:%.*]] = or <2 x i64> [[TMP42]], splat (i64 1)
; CHECK-NEXT: [[TMP41:%.*]] = shl <2 x i64> [[TMP42]], splat (i64 1)
; CHECK-NEXT: [[TMP39:%.*]] = mul i64 [[TMP0]], [[TMP0]]
; CHECK-NEXT: [[TMP43:%.*]] = add <8 x i64> [[TMP35]], [[TMP25]]
; CHECK-NEXT: [[TMP44:%.*]] = insertelement <64 x i64> [[TMP80]], i64 [[INDVAR37888]], i32 1
-; CHECK-NEXT: [[TMP45:%.*]] = insertelement <64 x i64> [[TMP44]], i64 [[TMP27]], i32 2
-; CHECK-NEXT: [[TMP46:%.*]] = insertelement <64 x i64> [[TMP45]], i64 [[TMP30]], i32 3
+; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <64 x i64> [[TMP44]], <64 x i64> [[TMP90]], <64 x i32> <i32 poison, i32 1, i32 64, i32 65, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 27, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <8 x i64> [[TMP35]], <8 x i64> poison, <64 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP48:%.*]] = shufflevector <4 x i64> [[TMP34]], <4 x i64> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP49:%.*]] = shufflevector <64 x i64> [[TMP46]], <64 x i64> [[TMP48]], <64 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 11, i32 poison, i32 65, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 66, i32 27, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/shuffled-gather-casted.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/shuffled-gather-casted.ll
index fc805b226d3b7..06c4bc205adf0 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/shuffled-gather-casted.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/shuffled-gather-casted.ll
@@ -59,12 +59,20 @@ define i32 @test1(ptr %p) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[D_0:%.*]] = load i16, ptr [[P]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i16> <i16 poison, i16 0, i16 0, i16 0>, i16 [[D_0]], i32 0
+; CHECK-NEXT: [[SZERO_2:%.*]] = sext i16 -1 to i32
+; CHECK-NEXT: [[UZERO_1:%.*]] = zext i16 -1 to i32
+; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i16> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[UZERO_1]], i32 2
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[SZERO_2]], i32 3
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> <i32 -1, i32 -16383, i32 undef, i32 undef>, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = zext <4 x i16> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP13]], [[TMP12]]
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[TMP0]] to <4 x i32>
-; CHECK-NEXT: [[TMP2:%.*]] = or <4 x i32> [[TMP3]], zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP2]], <i32 -1, i32 -16383, i32 65535, i32 -1>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> <i32 -1, i32 -16383, i32 65535, i32 -1>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP12]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[TMP4]], <i32 65535, i32 -16383, i32 65535, i32 65535>
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> <i32 4, i32 3, i32 2, i32 1>
+; CHECK-NEXT: [[TMP10:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i16>
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP5]], <4 x i16> [[TMP10]], <4 x i16> <i16 4, i16 3, i16 2, i16 1>
+; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i16> [[TMP11]] to <4 x i32>
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
; CHECK-NEXT: ret i32 [[TMP8]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll b/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
index f2ea2df7cc982..78fc3a60f0514 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
@@ -1,31 +1,17 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=x86_64-w64-windows-gnu < %s | FileCheck %s
-; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=x86_64-w64-windows-gnu\
-; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
define void @test(i16 %0) {
-; FORCED-LABEL: @test(
-; FORCED-NEXT: for.body92.preheader:
-; FORCED-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> <i16 0, i16 poison>, i16 [[TMP0:%.*]], i32 1
-; FORCED-NEXT: [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i32>
-; FORCED-NEXT: [[TMP3:%.*]] = zext <2 x i16> [[TMP1]] to <2 x i32>
-; FORCED-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; FORCED-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
-; FORCED-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>, <4 x i32> [[TMP5]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
-; FORCED-NEXT: br label [[FOR_BODY92:%.*]]
-; FORCED: for.body92:
-; FORCED-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> zeroinitializer, [[TMP6]]
-; FORCED-NEXT: store <4 x i32> [[TMP7]], ptr undef, align 8
-; FORCED-NEXT: br label [[FOR_BODY92]]
-;
; CHECK-LABEL: @test(
; CHECK-NEXT: for.body92.preheader:
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> <i16 0, i16 poison>, i16 [[TMP0:%.*]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = zext <2 x i16> [[TMP1]] to <2 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>, <4 x i32> [[TMP5]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
; CHECK-NEXT: br label [[FOR_BODY92:%.*]]
; CHECK: for.body92:
-; CHECK-NEXT: [[CONV177_I:%.*]] = sext i16 0 to i32
-; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>, i32 [[CONV177_I]], i32 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[TMP1]], i32 2
; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> zeroinitializer, [[TMP6]]
; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr undef, align 8
; CHECK-NEXT: br label [[FOR_BODY92]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
index 5a9ea0d292fa0..7ce45b872fae4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
@@ -1,7 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
; RUN: opt -passes=slp-vectorizer -S -slp-threshold=-9999 < %s | FileCheck %s
-; RUN: opt -passes=slp-vectorizer -S -slp-threshold=-9999\
-; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
define i64 @foo() {
; CHECK-LABEL: define i64 @foo() {
@@ -23,25 +21,6 @@ define i64 @foo() {
; CHECK-NEXT: [[ICMP:%.*]] = icmp ult i64 [[OR]], 0
; CHECK-NEXT: br i1 false, label [[BB3]], label [[BB1:%.*]]
;
-; FORCED-LABEL: define i64 @foo() {
-; FORCED-NEXT: bb:
-; FORCED-NEXT: [[ADD7:%.*]] = add i64 0, 0
-; FORCED-NEXT: br label [[BB3:%.*]]
-; FORCED: bb1:
-; FORCED-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ [[TMP5:%.*]], [[BB3]] ]
-; FORCED-NEXT: ret i64 0
-; FORCED: bb3:
-; FORCED-NEXT: [[PHI5:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ 0, [[BB3]] ]
-; FORCED-NEXT: [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, [[BB]] ], [ [[TMP7:%.*]], [[BB3]] ]
-; FORCED-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> <i64 poison, i64 0>, <2 x i32> <i32 0, i32 3>
-; FORCED-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[PHI5]], i32 0
-; FORCED-NEXT: [[TMP7]] = add <2 x i64> [[TMP6]], [[TMP2]]
-; FORCED-NEXT: [[TMP5]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; FORCED-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ADD7]]
-; FORCED-NEXT: [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
-; FORCED-NEXT: [[ICMP:%.*]] = icmp ult i64 [[TMP8]], 0
-; FORCED-NEXT: br i1 false, label [[BB3]], label [[BB1:%.*]]
-;
bb:
br label %bb3
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
index c5442b7fb7f13..6e656ba942a6b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
@@ -1,7 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-9999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
-; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-9999 -mtriple=x86_64-unknown-linux-gnu\
-; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
define void @foo() {
; CHECK-LABEL: define void @foo() {
@@ -21,23 +19,6 @@ define void @foo() {
; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ [[TMP2]], [[BB4]] ]
; CHECK-NEXT: ret void
;
-; FORCED-LABEL: define void @foo() {
-; FORCED-NEXT: bb:
-; FORCED-NEXT: br label [[BB1:%.*]]
-; FORCED: bb1:
-; FORCED-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
-; FORCED-NEXT: [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], zeroinitializer
-; FORCED-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
-; FORCED-NEXT: [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
-; FORCED-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
-; FORCED-NEXT: [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
-; FORCED-NEXT: br label [[BB4]]
-; FORCED: bb4:
-; FORCED-NEXT: br i1 false, label [[BB5:%.*]], label [[BB1]]
-; FORCED: bb5:
-; FORCED-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ [[TMP2]], [[BB4]] ]
-; FORCED-NEXT: ret void
-;
bb:
br label %bb1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_with_external_users.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_with_external_users.ll
index 93258f2975f34..098a2cd02caed 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_with_external_users.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_with_external_users.ll
@@ -160,9 +160,9 @@ define void @subadd_and_external_users(ptr %A, ptr %ptr) {
; CHECK-NEXT: [[LD:%.*]] = load double, ptr undef, align 8
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[LD]], i32 0
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = fsub <2 x double> [[TMP1]], <double 1.200000e+00, double 1.100000e+00>
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[TMP1]], <double 1.200000e+00, double 1.100000e+00>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP6]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP10:%.*]] = fsub <2 x double> [[TMP1]], <double 1.200000e+00, double 1.100000e+00>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[TMP4:%.*]] = fdiv <2 x double> [[TMP3]], <double 2.200000e+00, double 2.100000e+00>
; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP4]], <double 3.200000e+00, double 3.100000e+00>
; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[A:%.*]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shl-to-add-transformation5.ll b/llvm/test/Transforms/SLPVectorizer/X86/shl-to-add-transformation5.ll
index 6fea312b99b25..8f29f3f8de460 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shl-to-add-transformation5.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shl-to-add-transformation5.ll
@@ -7,10 +7,13 @@ define i32 @test(i32 %0, i32 %1) {
; CHECK-LABEL: define i32 @test(
; CHECK-SAME: i32 [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[ADD110:%.*]] = add i32 [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[DOTNEG_NEG:%.*]] = shl i32 [[TMP0]], 1
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i32 0
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = add <2 x i32> [[TMP3]], [[TMP12]]
+; CHECK-NEXT: [[TMP21:%.*]] = shl <2 x i32> [[TMP3]], [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP20]], <2 x i32> [[TMP21]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[DOTNEG_NEG:%.*]] = shl i32 [[TMP0]], 1
; CHECK-NEXT: [[TMP4:%.*]] = shl <2 x i32> [[TMP3]], <i32 0, i32 1>
; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @st, i64 12), align 4
; CHECK-NEXT: [[TMP6:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @st, i64 8), align 8
@@ -21,8 +24,6 @@ define i32 @test(i32 %0, i32 %1) {
; CHECK-NEXT: [[SUB120_3:%.*]] = or i32 [[TMP5]], [[DOTNEG_NEG]]
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> <i32 1, i32 poison, i32 1, i32 1>, i32 [[TMP0]], i32 1
; CHECK-NEXT: [[TMP11:%.*]] = shl <4 x i32> [[TMP10]], <i32 0, i32 1, i32 0, i32 0>
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> poison, i32 [[ADD110]], i32 0
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[DOTNEG_NEG]], i32 1
; CHECK-NEXT: [[TMP14:%.*]] = sub <2 x i32> zeroinitializer, [[TMP13]]
; CHECK-NEXT: store <2 x i32> [[TMP14]], ptr getelementptr inbounds nuw (i8, ptr @st, i64 32), align 16
; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> <i32 poison, i32 0, i32 poison, i32 poison>, <4 x i32> <i32 1, i32 5, i32 1, i32 poison>
@@ -65,11 +66,11 @@ define i32 @test1(ptr %0, ptr %1, i32 %2) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[ADD53_1:%.*]] = add i32 [[TMP4]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[ADD53_1]], i32 0
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i32 1
+; CHECK-NEXT: [[ADD53_1:%.*]] = add i32 [[TMP4]], [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP3]], i32 0
; CHECK-NEXT: [[TMP8:%.*]] = add <2 x i32> [[TMP7]], splat (i32 1)
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[ADD53_1]], i32 0
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP8]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[TMP10:%.*]] = sub <2 x i32> [[TMP5]], [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP10]], i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
index 1c4de256468c7..d6edf69882e82 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
@@ -7,14 +7,14 @@ define void @rftbsub(ptr %a) {
; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 2
; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 2, 1
; CHECK-NEXT: [[ARRAYIDX12:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT: [[SUB22:%.*]] = fsub double undef, undef
; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[ARRAYIDX12]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[ARRAYIDX6]], align 8
; CHECK-NEXT: [[ADD16:%.*]] = fadd double [[TMP1]], undef
; CHECK-NEXT: [[MUL18:%.*]] = fmul double undef, [[ADD16]]
-; CHECK-NEXT: [[ADD19:%.*]] = fadd double undef, [[MUL18]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[ADD19]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[SUB22]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> <double poison, double undef>, double [[MUL18]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> undef, [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = fsub <2 x double> undef, [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP7]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[TMP5:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[ARRAYIDX6]], align 8
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/small-graph-diff-block-instructions.ll b/llvm/test/Transforms/SLPVectorizer/X86/small-graph-diff-block-instructions.ll
index 82c8b1d707cf4..62c40cd5810bb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/small-graph-diff-block-instructions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/small-graph-diff-block-instructions.ll
@@ -6,12 +6,17 @@ define i32 @test(i32 %arg, i32 %arg1) {
; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[BB:.*:]]
; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[ARG1]] to i64
-; CHECK-NEXT: [[ZEXT2:%.*]] = zext i32 [[ARG]] to i64
-; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[ARG]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[ARG]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = sext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: br label %[[BB3:.*]]
; CHECK: [[BB3]]:
; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ZEXT]]
+; CHECK-NEXT: [[ZEXT2:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ZEXT2]]
+; CHECK-NEXT: [[SEXT:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
; CHECK-NEXT: [[GETELEMENTPTR5:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[SEXT]]
; CHECK-NEXT: [[ZEXT6:%.*]] = zext i32 0 to i64
; CHECK-NEXT: [[GETELEMENTPTR7:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ZEXT6]]
diff --git a/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll b/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll
index b23da5fa263f6..39c7602c95828 100644
--- a/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll
+++ b/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll
@@ -32,11 +32,12 @@ define <2 x float> @test_frem(float %a, i1 %cmp) {
define <2 x float> @replace_through_casts(i16 %inp) {
; CHECK-LABEL: define <2 x float> @replace_through_casts(
; CHECK-SAME: i16 [[INP:%.*]]) {
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10
-; CHECK-NEXT: [[TMP1:%.*]] = uitofp i16 [[INP]] to float
-; CHECK-NEXT: [[TMP2:%.*]] = sitofp i16 [[ADD]] to float
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i64 0
-; CHECK-NEXT: [[R:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[INP]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i16> [[TMP2]], <i16 0, i16 -10>
+; CHECK-NEXT: [[TMP4:%.*]] = uitofp <2 x i16> [[TMP3]] to <2 x float>
+; CHECK-NEXT: [[TMP5:%.*]] = sitofp <2 x i16> [[TMP3]] to <2 x float>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: ret <2 x float> [[R]]
;
%add = add nsw i16 %inp, -10
@@ -50,10 +51,11 @@ define <2 x float> @replace_through_casts(i16 %inp) {
define <2 x float> @replace_through_casts_and_binop(i16 %inp) {
; CHECK-LABEL: define <2 x float> @replace_through_casts_and_binop(
; CHECK-SAME: i16 [[INP:%.*]]) {
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10
-; CHECK-NEXT: [[MUL:%.*]] = mul nsw i16 [[INP]], 5
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[MUL]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i16> [[TMP1]], i16 [[ADD]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[INP]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = mul nsw <2 x i16> [[TMP6]], <i16 5, i16 -10>
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <2 x i16> [[TMP6]], <i16 5, i16 -10>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP7]], <2 x i16> [[TMP8]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[TMP3:%.*]] = uitofp <2 x i16> [[TMP2]] to <2 x float>
; CHECK-NEXT: [[TMP4:%.*]] = sitofp <2 x i16> [[TMP2]] to <2 x float>
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x i32> <i32 0, i32 3>
@@ -117,11 +119,12 @@ define <2 x float> @replace_through_casts_through_splat(i16 %inp) {
define <2 x i32> @replace_through_int_casts(i16 %inp, <2 x i16> %dead) {
; CHECK-LABEL: define <2 x i32> @replace_through_int_casts(
; CHECK-SAME: i16 [[INP:%.*]], <2 x i16> [[DEAD:%.*]]) {
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10
-; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[INP]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = sext i16 [[ADD]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i64 0
-; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[INP]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i16> [[TMP2]], <i16 0, i16 -10>
+; CHECK-NEXT: [[TMP4:%.*]] = zext <2 x i16> [[TMP3]] to <2 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = sext <2 x i16> [[TMP3]] to <2 x i32>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: ret <2 x i32> [[R]]
;
%add = add nsw i16 %inp, -10
@@ -135,10 +138,11 @@ define <2 x i32> @replace_through_int_casts(i16 %inp, <2 x i16> %dead) {
define <2 x i32> @replace_through_int_casts_ele0_only(i16 %inp, <2 x i16> %dead) {
; CHECK-LABEL: define <2 x i32> @replace_through_int_casts_ele0_only(
; CHECK-SAME: i16 [[INP:%.*]], <2 x i16> [[DEAD:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = sext i16 [[INP]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[INP]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i64 0
-; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[INP]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: ret <2 x i32> [[R]]
;
%2 = sext i16 %inp to i32
@@ -171,11 +175,12 @@ define <2 x i8> @replace_through_binop_fail_cant_speculate(i8 %inp, <2 x i8> %d,
define <2 x i8> @replace_through_binop_preserve_flags(i8 %inp, <2 x i8> %d, <2 x i8> %any) {
; CHECK-LABEL: define <2 x i8> @replace_through_binop_preserve_flags(
; CHECK-SAME: i8 [[INP:%.*]], <2 x i8> [[D:%.*]], <2 x i8> [[ANY:%.*]]) {
-; CHECK-NEXT: [[ADD:%.*]] = xor i8 [[INP]], 5
-; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[INP]], 123
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw i8 [[ADD]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i8> poison, i8 [[TMP1]], i64 0
-; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i8> [[TMP3]], i8 [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> poison, i8 [[INP]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i8> [[TMP2]], <i8 0, i8 5>
+; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i8> [[TMP3]], <i8 123, i8 1>
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <2 x i8> [[TMP3]], <i8 123, i8 1>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> [[TMP5]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: ret <2 x i8> [[R]]
;
%add = xor i8 %inp, 5
diff --git a/llvm/test/Transforms/SLPVectorizer/ext-int-reduced-not-operand.ll b/llvm/test/Transforms/SLPVectorizer/ext-int-reduced-not-operand.ll
index d8021538252c8..b1c1623b070d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/ext-int-reduced-not-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/ext-int-reduced-not-operand.ll
@@ -1,29 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-99999 < %s | FileCheck %s %}
-; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-99999\
-; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED %}
; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -slp-threshold=-99999 < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -slp-threshold=-99999\
-; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED %}
define i64 @wombat() {
-; FORCED-LABEL: define i64 @wombat() {
-; FORCED-NEXT: bb:
-; FORCED-NEXT: br label [[BB2:%.*]]
-; FORCED: bb1:
-; FORCED-NEXT: br label [[BB2]]
-; FORCED: bb2:
-; FORCED-NEXT: [[PHI:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ 0, [[BB1:%.*]] ]
-; FORCED-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[PHI]], i32 0
-; FORCED-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
-; FORCED-NEXT: [[TMP2:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i1>
-; FORCED-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; FORCED-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64
-; FORCED-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
-; FORCED-NEXT: [[TMP6:%.*]] = zext i1 [[TMP5]] to i64
-; FORCED-NEXT: [[OR:%.*]] = or i64 [[TMP4]], [[TMP6]]
-; FORCED-NEXT: ret i64 [[OR]]
-;
; CHECK-LABEL: define i64 @wombat() {
; CHECK-NEXT: bb:
; CHECK-NEXT: br label [[BB2:%.*]]
@@ -31,8 +10,13 @@ define i64 @wombat() {
; CHECK-NEXT: br label [[BB2]]
; CHECK: bb2:
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ 0, [[BB1:%.*]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[PHI]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[PHI]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[PHI]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = zext i1 [[TMP5]] to i64
; CHECK-NEXT: [[OR:%.*]] = or i64 [[TMP4]], [[TMP6]]
; CHECK-NEXT: ret i64 [[OR]]
;
>From f1fc7e282c87e9c01330c79765dcde0792142598 Mon Sep 17 00:00:00 2001
From: xlauko <xlauko at mail.muni.cz>
Date: Sat, 7 Mar 2026 08:42:26 +0100
Subject: [PATCH 24/25] [CIR] Add Commutative/Idempotent traits to binary ops
Add missing MLIR traits to CIR binary operations, matching the arith
dialect conventions:
- AndOp, OrOp: Commutative, Idempotent (fixes FIXME)
- AddOp, MulOp, XorOp, MaxOp: Commutative
Add these ops to the CIRCanonicalize pass op list so trait-based
folding is exercised by applyOpPatternsGreedily.
Update testFloatingPointBinOps in binop.cpp to use computed values,
preventing DCE of the now-canonicalized ops.
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 22 ++++---
.../Dialect/Transforms/CIRCanonicalize.cpp | 11 ++--
clang/test/CIR/CodeGen/binop.cpp | 46 ++++++++-----
clang/test/CIR/CodeGen/new.cpp | 9 +--
clang/test/CIR/CodeGen/size-of-vla.cpp | 10 +--
clang/test/CIR/CodeGen/throws.cpp | 4 +-
clang/test/CIR/CodeGen/vla.c | 8 +--
clang/test/CIR/Transforms/binop-traits.cir | 65 +++++++++++++++++++
8 files changed, 125 insertions(+), 50 deletions(-)
create mode 100644 clang/test/CIR/Transforms/binop-traits.cir
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 7f52be17d30f8..8860d6df27a74 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -2186,7 +2186,9 @@ class CIR_BinaryOpWithOverflowFlags<string mnemonic, Type type,
// AddOp
//===----------------------------------------------------------------------===//
-def CIR_AddOp : CIR_BinaryOpWithOverflowFlags<"add", CIR_AnyArithType> {
+def CIR_AddOp : CIR_BinaryOpWithOverflowFlags<"add", CIR_AnyArithType, [
+ Commutative
+]> {
let summary = "Integer or floating-point addition";
let description = [{
The `cir.add` operation performs addition on integer or floating-point
@@ -2241,7 +2243,7 @@ def CIR_SubOp : CIR_BinaryOpWithOverflowFlags<"sub", CIR_AnyArithType> {
// MulOp
//===----------------------------------------------------------------------===//
-def CIR_MulOp : CIR_BinaryOp<"mul", CIR_AnyArithType> {
+def CIR_MulOp : CIR_BinaryOp<"mul", CIR_AnyArithType, [Commutative]> {
let summary = "Integer or floating-point multiplication";
let description = [{
The `cir.mul` operation performs multiplication on integer or floating-point
@@ -2321,8 +2323,9 @@ def CIR_RemOp : CIR_BinaryOp<"rem", CIR_AnyArithType> {
// AndOp
//===----------------------------------------------------------------------===//
-// FIXME: Commutative, Idempotent traits
-def CIR_AndOp : CIR_BinaryOp<"and", CIR_AnyBitwiseType> {
+def CIR_AndOp : CIR_BinaryOp<"and", CIR_AnyBitwiseType, [
+ Commutative, Idempotent
+]> {
let summary = "Bitwise AND";
let description = [{
The `cir.and` operation performs a bitwise AND on integer operands.
@@ -2341,8 +2344,9 @@ def CIR_AndOp : CIR_BinaryOp<"and", CIR_AnyBitwiseType> {
// OrOp
//===----------------------------------------------------------------------===//
-// FIXME: Commutative, Idempotent traits
-def CIR_OrOp : CIR_BinaryOp<"or", CIR_AnyBitwiseType> {
+def CIR_OrOp : CIR_BinaryOp<"or", CIR_AnyBitwiseType, [
+ Commutative, Idempotent
+]> {
let summary = "Bitwise OR";
let description = [{
The `cir.or` operation performs a bitwise OR on integer operands.
@@ -2361,7 +2365,7 @@ def CIR_OrOp : CIR_BinaryOp<"or", CIR_AnyBitwiseType> {
// XorOp
//===----------------------------------------------------------------------===//
-def CIR_XorOp : CIR_BinaryOp<"xor", CIR_AnyBitwiseType> {
+def CIR_XorOp : CIR_BinaryOp<"xor", CIR_AnyBitwiseType, [Commutative]> {
let summary = "Bitwise XOR";
let description = [{
The `cir.xor` operation performs a bitwise XOR on integer operands.
@@ -2380,7 +2384,9 @@ def CIR_XorOp : CIR_BinaryOp<"xor", CIR_AnyBitwiseType> {
// MaxOp
//===----------------------------------------------------------------------===//
-def CIR_MaxOp : CIR_BinaryOp<"max", CIR_AnyIntOrVecOfIntType> {
+def CIR_MaxOp : CIR_BinaryOp<"max", CIR_AnyIntOrVecOfIntType, [
+ Commutative, Idempotent
+]> {
let summary = "Integer maximum";
let description = [{
The `cir.max` operation computes the maximum of two integer operands.
diff --git a/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp b/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
index e9b825606a04e..46a8c011b320b 100644
--- a/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
@@ -70,11 +70,12 @@ void CIRCanonicalizePass::runOnOperation() {
// Many operations are here to perform a manual `fold` in
// applyOpPatternsGreedily.
- if (isa<BrOp, BrCondOp, CastOp, ScopeOp, SwitchOp, SelectOp, UnaryOp,
- ComplexCreateOp, ComplexImagOp, ComplexRealOp, VecCmpOp,
- VecCreateOp, VecExtractOp, VecShuffleOp, VecShuffleDynamicOp,
- VecTernaryOp, BitClrsbOp, BitClzOp, BitCtzOp, BitFfsOp, BitParityOp,
- BitPopcountOp, BitReverseOp, ByteSwapOp, RotateOp, ConstantOp>(op))
+ if (isa<BrOp, BrCondOp, CastOp, ScopeOp, SwitchOp, SelectOp, UnaryOp, AddOp,
+ MulOp, AndOp, OrOp, XorOp, MaxOp, ComplexCreateOp, ComplexImagOp,
+ ComplexRealOp, VecCmpOp, VecCreateOp, VecExtractOp, VecShuffleOp,
+ VecShuffleDynamicOp, VecTernaryOp, BitClrsbOp, BitClzOp, BitCtzOp,
+ BitFfsOp, BitParityOp, BitPopcountOp, BitReverseOp, ByteSwapOp,
+ RotateOp, ConstantOp>(op))
ops.push_back(op);
});
diff --git a/clang/test/CIR/CodeGen/binop.cpp b/clang/test/CIR/CodeGen/binop.cpp
index 3d883f14acdc9..33498b90e3a42 100644
--- a/clang/test/CIR/CodeGen/binop.cpp
+++ b/clang/test/CIR/CodeGen/binop.cpp
@@ -127,10 +127,10 @@ void b0(int a, int b) {
// OGCG: ret void
void testFloatingPointBinOps(float a, float b) {
- a * b;
- a / b;
- a + b;
- a - b;
+ float x = a * b;
+ x = x / b;
+ x = x + b;
+ x = x - b;
}
// CIR-LABEL: cir.func{{.*}} @_Z23testFloatingPointBinOpsff(
@@ -144,48 +144,58 @@ void testFloatingPointBinOps(float a, float b) {
// LLVM-SAME: float {{.*}} %[[A:.*]], float {{.*}} %[[B:.*]])
// LLVM: %[[A_ADDR:.*]] = alloca float, i64 1
// LLVM: %[[B_ADDR:.*]] = alloca float, i64 1
+// LLVM: %[[X_ADDR:.*]] = alloca float, i64 1
// LLVM: store float %[[A]], ptr %[[A_ADDR]]
// LLVM: store float %[[B]], ptr %[[B_ADDR]]
// LLVM: %[[A1:.*]] = load float, ptr %[[A_ADDR]]
// LLVM: %[[B1:.*]] = load float, ptr %[[B_ADDR]]
-// LLVM: fmul float %[[A1]], %[[B1]]
+// LLVM: %[[MUL:.*]] = fmul float %[[A1]], %[[B1]]
+// LLVM: store float %[[MUL]], ptr %[[X_ADDR]]
-// LLVM: %[[A2:.*]] = load float, ptr %[[A_ADDR]]
+// LLVM: %[[X1:.*]] = load float, ptr %[[X_ADDR]]
// LLVM: %[[B2:.*]] = load float, ptr %[[B_ADDR]]
-// LLVM: fdiv float %[[A2]], %[[B2]]
+// LLVM: %[[DIV:.*]] = fdiv float %[[X1]], %[[B2]]
+// LLVM: store float %[[DIV]], ptr %[[X_ADDR]]
-// LLVM: %[[A3:.*]] = load float, ptr %[[A_ADDR]]
+// LLVM: %[[X2:.*]] = load float, ptr %[[X_ADDR]]
// LLVM: %[[B3:.*]] = load float, ptr %[[B_ADDR]]
-// LLVM: fadd float %[[A3]], %[[B3]]
+// LLVM: %[[ADD:.*]] = fadd float %[[X2]], %[[B3]]
+// LLVM: store float %[[ADD]], ptr %[[X_ADDR]]
-// LLVM: %[[A4:.*]] = load float, ptr %[[A_ADDR]]
+// LLVM: %[[X3:.*]] = load float, ptr %[[X_ADDR]]
// LLVM: %[[B4:.*]] = load float, ptr %[[B_ADDR]]
-// LLVM: fsub float %[[A4]], %[[B4]]
+// LLVM: %[[SUB:.*]] = fsub float %[[X3]], %[[B4]]
+// LLVM: store float %[[SUB]], ptr %[[X_ADDR]]
// LLVM: ret void
// OGCG-LABEL: define{{.*}} void @_Z23testFloatingPointBinOpsff(float {{.*}} %a, float {{.*}} %b)
// OGCG: %a.addr = alloca float
// OGCG: %b.addr = alloca float
+// OGCG: %x = alloca float
// OGCG: store float %a, ptr %a.addr
// OGCG: store float %b, ptr %b.addr
// OGCG: %[[A1:.*]] = load float, ptr %a.addr
// OGCG: %[[B1:.*]] = load float, ptr %b.addr
-// OGCG: fmul float %[[A1]], %[[B1]]
+// OGCG: %[[MUL:.*]] = fmul float %[[A1]], %[[B1]]
+// OGCG: store float %[[MUL]], ptr %x
-// OGCG: %[[A2:.*]] = load float, ptr %a.addr
+// OGCG: %[[X1:.*]] = load float, ptr %x
// OGCG: %[[B2:.*]] = load float, ptr %b.addr
-// OGCG: fdiv float %[[A2]], %[[B2]]
+// OGCG: %[[DIV:.*]] = fdiv float %[[X1]], %[[B2]]
+// OGCG: store float %[[DIV]], ptr %x
-// OGCG: %[[A3:.*]] = load float, ptr %a.addr
+// OGCG: %[[X2:.*]] = load float, ptr %x
// OGCG: %[[B3:.*]] = load float, ptr %b.addr
-// OGCG: fadd float %[[A3]], %[[B3]]
+// OGCG: %[[ADD:.*]] = fadd float %[[X2]], %[[B3]]
+// OGCG: store float %[[ADD]], ptr %x
-// OGCG: %[[A4:.*]] = load float, ptr %a.addr
+// OGCG: %[[X3:.*]] = load float, ptr %x
// OGCG: %[[B4:.*]] = load float, ptr %b.addr
-// OGCG: fsub float %[[A4]], %[[B4]]
+// OGCG: %[[SUB:.*]] = fsub float %[[X3]], %[[B4]]
+// OGCG: store float %[[SUB]], ptr %x
// OGCG: ret void
diff --git a/clang/test/CIR/CodeGen/new.cpp b/clang/test/CIR/CodeGen/new.cpp
index 20fe26ac833e0..74aa74a8d51af 100644
--- a/clang/test/CIR/CodeGen/new.cpp
+++ b/clang/test/CIR/CodeGen/new.cpp
@@ -551,19 +551,14 @@ void t_new_var_size5(int n) {
auto p = new double[n][2][3];
}
-// NUM_ELEMENTS isn't used in this case because there is no cookie. It isn't
-// used in the allocation size because the allocation size is calculated with
-// the element size and the fixed size dimensions already combined (6 * 8 = 48).
-// We don't DCE NUM_ELEMENTS because it's not a constant, but later
-// optimizations will eliminate it.
+// The allocation size is calculated with the element size and the fixed size
+// dimensions already combined (6 * 8 = 48).
// CHECK: cir.func {{.*}} @_Z15t_new_var_size5i
// CHECK: %[[N:.*]] = cir.load{{.*}} %[[ARG_ALLOCA:.*]]
// CHECK: %[[N_SIZE_T:.*]] = cir.cast integral %[[N]] : !s32i -> !u64i
// CHECK: %[[ELEMENT_SIZE:.*]] = cir.const #cir.int<48> : !u64i
// CHECK: %[[RESULT:.*]], %[[OVERFLOW:.*]] = cir.binop.overflow(mul, %[[N_SIZE_T]], %[[ELEMENT_SIZE]]) : !u64i, (!u64i, !cir.bool)
-// CHECK: %[[NUM_ELEMENTS_MULTIPLIER:.*]] = cir.const #cir.int<6>
-// CHECK: %[[NUM_ELEMENTS:.*]] = cir.mul %[[N_SIZE_T]], %[[NUM_ELEMENTS_MULTIPLIER]] : !u64i
// CHECK: %[[ALL_ONES:.*]] = cir.const #cir.int<18446744073709551615> : !u64i
// CHECK: %[[ALLOC_SIZE:.*]] = cir.select if %[[OVERFLOW]] then %[[ALL_ONES]] else %[[RESULT]] : (!cir.bool, !u64i, !u64i)
// CHECK: %[[PTR:.*]] = cir.call @_Znam(%[[ALLOC_SIZE]]) {allocsize = array<i32: 0>} : (!u64i {llvm.noundef})
diff --git a/clang/test/CIR/CodeGen/size-of-vla.cpp b/clang/test/CIR/CodeGen/size-of-vla.cpp
index 789e97f40440b..3f9b2f5fecbbd 100644
--- a/clang/test/CIR/CodeGen/size-of-vla.cpp
+++ b/clang/test/CIR/CodeGen/size-of-vla.cpp
@@ -38,16 +38,16 @@ void vla_type_with_element_type_int() {
// CIR: %[[SIZE_ADDR:.*]] = cir.alloca !u64i, !cir.ptr<!u64i>, ["size", init]
// CIR: %[[CONST_10:.*]] = cir.const #cir.int<10> : !u64i
// CIR: cir.store {{.*}} %[[CONST_10]], %[[N_ADDR]] : !u64i, !cir.ptr<!u64i>
-// CIR: %3 = cir.load {{.*}} %[[N_ADDR]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[TMP_N:.*]] = cir.load {{.*}} %[[N_ADDR]] : !cir.ptr<!u64i>, !u64i
// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !u64i
-// CIR: %[[SIZE:.*]] = cir.mul nuw %[[CONST_4]], %3 : !u64i
+// CIR: %[[SIZE:.*]] = cir.mul nuw %[[TMP_N]], %[[CONST_4]] : !u64i
// CIR: cir.store {{.*}} %[[SIZE]], %[[SIZE_ADDR]] : !u64i, !cir.ptr<!u64i>
// LLVM: %[[N_ADDR:.*]] = alloca i64, i64 1, align 8
// LLVM: %[[SIZE_ADDR:.*]] = alloca i64, i64 1, align 8
// LLVM: store i64 10, ptr %[[N_ADDR]], align 8
// LLVM: %[[TMP_N:.*]] = load i64, ptr %[[N_ADDR]], align 8
-// LLVM: %[[SIZE:.*]] = mul nuw i64 4, %[[TMP_N]]
+// LLVM: %[[SIZE:.*]] = mul nuw i64 %[[TMP_N]], 4
// LLVM: store i64 %[[SIZE]], ptr %[[SIZE_ADDR]], align 8
// OGCG: %[[N_ADDR:.*]] = alloca i64, align 8
@@ -126,7 +126,7 @@ void vla_expr_element_type_int() {
// CIR: cir.cleanup.scope {
// CIR: %[[ARR_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, %[[TMP_N]] : !u64i, ["arr"]
// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !u64i
-// CIR: %[[SIZE:.*]] = cir.mul nuw %[[CONST_4]], %[[TMP_N]] : !u64i
+// CIR: %[[SIZE:.*]] = cir.mul nuw %[[TMP_N]], %[[CONST_4]] : !u64i
// CIR: cir.store {{.*}} %[[SIZE]], %[[SIZE_ADDR]] : !u64i, !cir.ptr<!u64i>
// CIR: cir.yield
// CIR: } cleanup normal {
@@ -143,7 +143,7 @@ void vla_expr_element_type_int() {
// LLVM: %[[STACK_SAVE:.*]] = call ptr @llvm.stacksave.p0()
// LLVM: store ptr %[[STACK_SAVE]], ptr %[[SAVED_STACK_ADDR]], align 8
// LLVM: %[[ARR_ADDR:.*]] = alloca i32, i64 %[[TMP_N]], align 16
-// LLVM: %[[SIZE:.*]] = mul nuw i64 4, %[[TMP_N]]
+// LLVM: %[[SIZE:.*]] = mul nuw i64 %[[TMP_N]], 4
// LLVM: store i64 %[[SIZE]], ptr %[[SIZE_ADDR]], align 8
// LLVM: %[[TMP_SAVED_STACK:.*]] = load ptr, ptr %[[SAVED_STACK_ADDR]], align 8
// LLVM: call void @llvm.stackrestore.p0(ptr %[[TMP_SAVED_STACK]])
diff --git a/clang/test/CIR/CodeGen/throws.cpp b/clang/test/CIR/CodeGen/throws.cpp
index 64cf0f6d62b0a..14b1d1272217c 100644
--- a/clang/test/CIR/CodeGen/throws.cpp
+++ b/clang/test/CIR/CodeGen/throws.cpp
@@ -112,9 +112,7 @@ void paren_expr() { (throw 0, 1 + 2); }
// CIR: cir.throw %[[EXCEPTION_ADDR]] : !cir.ptr<!s32i>, @_ZTIi
// CIR: cir.unreachable
// CIR: ^bb1:
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[ADD:.*]] = cir.add nsw %[[CONST_1]], %[[CONST_2]] : !s32i
+// CIR: cir.return
// LLVM: %[[EXCEPTION_ADDR:.*]] = call ptr @__cxa_allocate_exception(i64 4)
// LLVM: store i32 0, ptr %[[EXCEPTION_ADDR]], align 16
diff --git a/clang/test/CIR/CodeGen/vla.c b/clang/test/CIR/CodeGen/vla.c
index 2eb8d3c3814af..f30a438834660 100644
--- a/clang/test/CIR/CodeGen/vla.c
+++ b/clang/test/CIR/CodeGen/vla.c
@@ -62,7 +62,7 @@ void f1(int len) {
// CIR: %[[LEN_SIZE_T:.*]] = cir.cast integral %[[LEN]] : !s32i -> !u64i
// CIR: %[[STACK_PTR:.*]] = cir.stacksave
// CIR: cir.store{{.*}} %[[STACK_PTR]], %[[SAVED_STACK]]
-// CIR: %[[TOTAL_LEN:.*]] = cir.mul nuw %[[SIXTEEN]], %[[LEN_SIZE_T]]
+// CIR: %[[TOTAL_LEN:.*]] = cir.mul nuw %[[LEN_SIZE_T]], %[[SIXTEEN]]
// CIR: %[[ARR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, %[[TOTAL_LEN]] : !u64i, ["arr"]
// CIR: %[[STACK_RESTORE_PTR:.*]] = cir.load{{.*}} %[[SAVED_STACK]]
// CIR: cir.stackrestore %[[STACK_RESTORE_PTR]]
@@ -75,7 +75,7 @@ void f1(int len) {
// LLVM: %[[LEN_SIZE_T:.*]] = sext i32 %[[LEN]] to i64
// LLVM: %[[STACK_PTR:.*]] = call ptr @llvm.stacksave.p0()
// LLVM: store ptr %[[STACK_PTR]], ptr %[[SAVED_STACK]]
-// LLVM: %[[TOTAL_LEN:.*]] = mul nuw i64 16, %[[LEN_SIZE_T]]
+// LLVM: %[[TOTAL_LEN:.*]] = mul nuw i64 %[[LEN_SIZE_T]], 16
// LLVM: %[[ARR:.*]] = alloca i32, i64 %[[TOTAL_LEN]]
// LLVM: %[[STACK_RESTORE_PTR:.*]] = load ptr, ptr %[[SAVED_STACK]]
// LLVM: call void @llvm.stackrestore.p0(ptr %[[STACK_RESTORE_PTR]])
@@ -361,7 +361,7 @@ void vla_subscript_expr() {
// CIR: %[[COMPOUND_PTR:.*]] = cir.ptr_stride %[[TMP_COMPOUND]], %[[CONST_0]] : (!cir.ptr<!cir.ptr<!s32i>>, !s32i) -> !cir.ptr<!cir.ptr<!s32i>>
// CIR: %[[TMP_COMPOUND:.*]] = cir.load {{.*}} %10 : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !u64i
-// CIR: %[[VLA_IDX:.*]] = cir.mul nsw %[[CONST_1]], %7 : !u64i
+// CIR: %[[VLA_IDX:.*]] = cir.mul nsw %[[TMP_N]], %[[CONST_1]] : !u64i
// CIR: %[[VLA_A_PTR:.*]] = cir.ptr_stride %[[TMP_COMPOUND]], %[[VLA_IDX]] : (!cir.ptr<!s32i>, !u64i) -> !cir.ptr<!s32i>
// CIR: %[[ELEM_5_PTR:.*]] = cir.ptr_stride %[[VLA_A_PTR]], %[[CONST_5]] : (!cir.ptr<!s32i>, !s32i) -> !cir.ptr<!s32i>
// CIR: cir.store {{.*}} %[[CONST_0_VAL]], %[[ELEM_5_PTR]] : !s32i, !cir.ptr<!s32i>
@@ -375,7 +375,7 @@ void vla_subscript_expr() {
// LLVM: %[[TMP_COMPOUND:.*]] = load ptr, ptr %[[COMPOUND_ADDR]], align 8
// LLVM: %[[COMPOUND_PTR:.*]] = getelementptr ptr, ptr %[[TMP_COMPOUND]], i64 0
// LLVM: %[[TMP_COMPOUND:.*]] = load ptr, ptr %[[COMPOUND_PTR]], align 8
-// LLVM: %[[VLA_IDX:.*]] = mul nsw i64 1, %[[TMP_N]]
+// LLVM: %[[VLA_IDX:.*]] = mul nsw i64 %[[TMP_N]], 1
// LLVM: %[[VLA_A_PTR:.*]] = getelementptr i32, ptr %[[TMP_COMPOUND]], i64 %[[VLA_IDX]]
// LLVM: %[[ELEM_5_PTR:.*]] = getelementptr i32, ptr %[[VLA_A_PTR]], i64 5
// LLVM: store i32 0, ptr %[[ELEM_5_PTR]], align 4
diff --git a/clang/test/CIR/Transforms/binop-traits.cir b/clang/test/CIR/Transforms/binop-traits.cir
new file mode 100644
index 0000000000000..df2b12cf9f178
--- /dev/null
+++ b/clang/test/CIR/Transforms/binop-traits.cir
@@ -0,0 +1,65 @@
+// RUN: cir-opt %s -cir-canonicalize -o - | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+// CHECK-LABEL: @and_idempotent
+// CHECK-NEXT: cir.return %arg0
+cir.func @and_idempotent(%arg0 : !s32i) -> !s32i {
+ %0 = cir.and %arg0, %arg0 : !s32i
+ cir.return %0 : !s32i
+}
+
+// CHECK-LABEL: @or_idempotent
+// CHECK-NEXT: cir.return %arg0
+cir.func @or_idempotent(%arg0 : !s32i) -> !s32i {
+ %0 = cir.or %arg0, %arg0 : !s32i
+ cir.return %0 : !s32i
+}
+
+// CHECK-LABEL: @and_commutative
+// CHECK: cir.and %arg0, %{{.*}} : !s32i
+cir.func @and_commutative(%arg0 : !s32i) -> !s32i {
+ %0 = cir.const #cir.int<42> : !s32i
+ %1 = cir.and %0, %arg0 : !s32i
+ cir.return %1 : !s32i
+}
+
+// CHECK-LABEL: @or_commutative
+// CHECK: cir.or %arg0, %{{.*}} : !s32i
+cir.func @or_commutative(%arg0 : !s32i) -> !s32i {
+ %0 = cir.const #cir.int<42> : !s32i
+ %1 = cir.or %0, %arg0 : !s32i
+ cir.return %1 : !s32i
+}
+
+// CHECK-LABEL: @xor_commutative
+// CHECK: cir.xor %arg0, %{{.*}} : !s32i
+cir.func @xor_commutative(%arg0 : !s32i) -> !s32i {
+ %0 = cir.const #cir.int<42> : !s32i
+ %1 = cir.xor %0, %arg0 : !s32i
+ cir.return %1 : !s32i
+}
+
+// CHECK-LABEL: @add_commutative
+// CHECK: cir.add %arg0, %{{.*}} : !s32i
+cir.func @add_commutative(%arg0 : !s32i) -> !s32i {
+ %0 = cir.const #cir.int<42> : !s32i
+ %1 = cir.add %0, %arg0 : !s32i
+ cir.return %1 : !s32i
+}
+
+// CHECK-LABEL: @mul_commutative
+// CHECK: cir.mul %arg0, %{{.*}} : !s32i
+cir.func @mul_commutative(%arg0 : !s32i) -> !s32i {
+ %0 = cir.const #cir.int<42> : !s32i
+ %1 = cir.mul %0, %arg0 : !s32i
+ cir.return %1 : !s32i
+}
+
+// CHECK-LABEL: @max_commutative
+// CHECK: cir.max %arg0, %{{.*}} : !s32i
+cir.func @max_commutative(%arg0 : !s32i) -> !s32i {
+ %0 = cir.const #cir.int<42> : !s32i
+ %1 = cir.max %0, %arg0 : !s32i
+ cir.return %1 : !s32i
+}
>From 06a94e24ede6c141de0fd8351ac29576cc9015ed Mon Sep 17 00:00:00 2001
From: xlauko <xlauko at mail.muni.cz>
Date: Sat, 7 Mar 2026 14:19:44 +0100
Subject: [PATCH 25/25] [CIR] Remove cir.unary(plus, ...) and emit nothing for
unary plus
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Traditional codegen never emits any operation for unary plus — it just
visits the subexpression as a pure identity at the codegen level. Align
CIRGen with this behavior by removing Plus from UnaryOpKind entirely
and having VisitUnaryPlus directly visit the subexpression with the
appropriate promotion/demotion handling.
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 7 +-
clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp | 34 ++---
clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp | 26 ++--
clang/lib/CIR/Dialect/IR/CIRDialect.cpp | 11 +-
.../Dialect/Transforms/LoweringPrepare.cpp | 1 -
.../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 24 +---
clang/lib/CodeGen/CGExprComplex.cpp | 135 +++++++++---------
clang/test/CIR/CodeGen/complex-unary.cpp | 36 ++---
clang/test/CIR/CodeGen/complex.cpp | 20 +--
clang/test/CIR/CodeGen/coro-task.cpp | 3 +-
.../CIR/CodeGen/lambda-static-invoker.cpp | 3 +-
clang/test/CIR/CodeGen/unary.cpp | 15 +-
clang/test/CIR/CodeGen/vector-ext.cpp | 3 +-
clang/test/CIR/CodeGen/vector.cpp | 3 +-
clang/test/CIR/IR/unary.cir | 36 +++--
clang/test/CIR/Transforms/canonicalize.cir | 72 ----------
16 files changed, 146 insertions(+), 283 deletions(-)
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 8860d6df27a74..7656ee4bbb0d5 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -1709,16 +1709,15 @@ def CIR_LabelOp : CIR_Op<"label", [AlwaysSpeculatable]> {
def CIR_UnaryOpKind : CIR_I32EnumAttr<"UnaryOpKind", "unary operation kind", [
I32EnumAttrCase<"Inc", 0, "inc">,
I32EnumAttrCase<"Dec", 1, "dec">,
- I32EnumAttrCase<"Plus", 2, "plus">,
- I32EnumAttrCase<"Minus", 3, "minus">,
- I32EnumAttrCase<"Not", 4, "not">
+ I32EnumAttrCase<"Minus", 2, "minus">,
+ I32EnumAttrCase<"Not", 3, "not">
]>;
def CIR_UnaryOp : CIR_Op<"unary", [Pure, SameOperandsAndResultType]> {
let summary = "Unary operations";
let description = [{
`cir.unary` performs the unary operation according to
- the specified opcode kind: [inc, dec, plus, minus, not].
+ the specified opcode kind: [inc, dec, minus, not].
It requires one input operand and has one result, both types
should be the same.
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index ff5a58a8eb122..1d05f03491d40 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -193,9 +193,9 @@ class ComplexExprEmitter : public StmtVisitor<ComplexExprEmitter, mlir::Value> {
mlir::Value VisitUnaryDeref(const Expr *e) { return emitLoadOfLValue(e); }
mlir::Value VisitUnaryPlus(const UnaryOperator *e);
+ mlir::Value VisitUnaryPlus(const UnaryOperator *e, QualType promotionType);
mlir::Value VisitUnaryMinus(const UnaryOperator *e);
- mlir::Value VisitPlusMinus(const UnaryOperator *e, cir::UnaryOpKind kind,
- QualType promotionType);
+ mlir::Value VisitUnaryMinus(const UnaryOperator *e, QualType promotionType);
mlir::Value VisitUnaryNot(const UnaryOperator *e);
// LNot,Real,Imag never return complex.
mlir::Value VisitUnaryExtension(const UnaryOperator *e) {
@@ -574,32 +574,36 @@ mlir::Value ComplexExprEmitter::emitCast(CastKind ck, Expr *op,
mlir::Value ComplexExprEmitter::VisitUnaryPlus(const UnaryOperator *e) {
QualType promotionTy = getPromotionType(e->getSubExpr()->getType());
- mlir::Value result = VisitPlusMinus(e, cir::UnaryOpKind::Plus, promotionTy);
+ mlir::Value result = VisitUnaryPlus(e, promotionTy);
if (!promotionTy.isNull())
return cgf.emitUnPromotedValue(result, e->getSubExpr()->getType());
return result;
}
+mlir::Value ComplexExprEmitter::VisitUnaryPlus(const UnaryOperator *e,
+ QualType promotionType) {
+ if (!promotionType.isNull())
+ return cgf.emitPromotedComplexExpr(e->getSubExpr(), promotionType);
+ return Visit(e->getSubExpr());
+}
+
mlir::Value ComplexExprEmitter::VisitUnaryMinus(const UnaryOperator *e) {
QualType promotionTy = getPromotionType(e->getSubExpr()->getType());
- mlir::Value result = VisitPlusMinus(e, cir::UnaryOpKind::Minus, promotionTy);
+ mlir::Value result = VisitUnaryMinus(e, promotionTy);
if (!promotionTy.isNull())
return cgf.emitUnPromotedValue(result, e->getSubExpr()->getType());
return result;
}
-mlir::Value ComplexExprEmitter::VisitPlusMinus(const UnaryOperator *e,
- cir::UnaryOpKind kind,
- QualType promotionType) {
- assert((kind == cir::UnaryOpKind::Plus || kind == cir::UnaryOpKind::Minus) &&
- "Invalid UnaryOp kind for ComplexType Plus or Minus");
-
+mlir::Value ComplexExprEmitter::VisitUnaryMinus(const UnaryOperator *e,
+ QualType promotionType) {
mlir::Value op;
if (!promotionType.isNull())
op = cgf.emitPromotedComplexExpr(e->getSubExpr(), promotionType);
else
op = Visit(e->getSubExpr());
- return builder.createUnaryOp(cgf.getLoc(e->getExprLoc()), kind, op);
+ return builder.createUnaryOp(cgf.getLoc(e->getExprLoc()),
+ cir::UnaryOpKind::Minus, op);
}
mlir::Value ComplexExprEmitter::VisitUnaryNot(const UnaryOperator *e) {
@@ -766,12 +770,10 @@ mlir::Value ComplexExprEmitter::emitPromoted(const Expr *e,
}
} else if (const auto *unaryOp = dyn_cast<UnaryOperator>(e)) {
switch (unaryOp->getOpcode()) {
+ case UO_Plus:
+ return VisitUnaryPlus(unaryOp, promotionTy);
case UO_Minus:
- case UO_Plus: {
- auto kind = unaryOp->getOpcode() == UO_Plus ? cir::UnaryOpKind::Plus
- : cir::UnaryOpKind::Minus;
- return VisitPlusMinus(unaryOp, kind, promotionTy);
- }
+ return VisitUnaryMinus(unaryOp, promotionTy);
default:
break;
}
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 1b5ba5ee6783f..29aa35e762f45 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -766,25 +766,28 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
mlir::Value VisitUnaryPlus(const UnaryOperator *e) {
QualType promotionType = getPromotionType(e->getSubExpr()->getType());
- mlir::Value result =
- emitUnaryPlusOrMinus(e, cir::UnaryOpKind::Plus, promotionType);
+ mlir::Value result = VisitUnaryPlus(e, promotionType);
if (result && !promotionType.isNull())
return emitUnPromotedValue(result, e->getType());
return result;
}
+ mlir::Value VisitUnaryPlus(const UnaryOperator *e, QualType promotionType) {
+ ignoreResultAssign = false;
+ if (!promotionType.isNull())
+ return cgf.emitPromotedScalarExpr(e->getSubExpr(), promotionType);
+ return Visit(e->getSubExpr());
+ }
+
mlir::Value VisitUnaryMinus(const UnaryOperator *e) {
QualType promotionType = getPromotionType(e->getSubExpr()->getType());
- mlir::Value result =
- emitUnaryPlusOrMinus(e, cir::UnaryOpKind::Minus, promotionType);
+ mlir::Value result = VisitUnaryMinus(e, promotionType);
if (result && !promotionType.isNull())
return emitUnPromotedValue(result, e->getType());
return result;
}
- mlir::Value emitUnaryPlusOrMinus(const UnaryOperator *e,
- cir::UnaryOpKind kind,
- QualType promotionType) {
+ mlir::Value VisitUnaryMinus(const UnaryOperator *e, QualType promotionType) {
ignoreResultAssign = false;
mlir::Value operand;
if (!promotionType.isNull())
@@ -795,14 +798,13 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
// TODO(cir): We might have to change this to support overflow trapping.
// Classic codegen routes unary minus through emitSub to ensure
// that the overflow behavior is handled correctly.
- bool nsw = kind == cir::UnaryOpKind::Minus &&
- e->getType()->isSignedIntegerType() &&
+ bool nsw = e->getType()->isSignedIntegerType() &&
cgf.getLangOpts().getSignedOverflowBehavior() !=
LangOptions::SOB_Defined;
// NOTE: LLVM codegen will lower this directly to either a FNeg
// or a Sub instruction. In CIR this will be handled later in LowerToLLVM.
- return emitUnaryOp(e, kind, operand, nsw);
+ return emitUnaryOp(e, cir::UnaryOpKind::Minus, operand, nsw);
}
mlir::Value emitUnaryOp(const UnaryOperator *e, cir::UnaryOpKind kind,
@@ -1560,9 +1562,9 @@ mlir::Value ScalarExprEmitter::emitPromoted(const Expr *e,
case UO_Real:
return VisitRealImag(uo, promotionType);
case UO_Minus:
- return emitUnaryPlusOrMinus(uo, cir::UnaryOpKind::Minus, promotionType);
+ return VisitUnaryMinus(uo, promotionType);
case UO_Plus:
- return emitUnaryPlusOrMinus(uo, cir::UnaryOpKind::Plus, promotionType);
+ return VisitUnaryPlus(uo, promotionType);
default:
break;
}
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 8d2990af5de8c..87cf3950b04d6 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -2671,7 +2671,6 @@ LogicalResult cir::UnaryOp::verify() {
switch (getKind()) {
case cir::UnaryOpKind::Inc:
case cir::UnaryOpKind::Dec:
- case cir::UnaryOpKind::Plus:
case cir::UnaryOpKind::Minus:
case cir::UnaryOpKind::Not:
// Nothing to verify.
@@ -2712,9 +2711,8 @@ OpFoldResult cir::UnaryOp::fold(FoldAdaptor adaptor) {
// input attribute from the adapter, a new constant is materialized, but
// if we return the input value directly, it avoids that.
if (auto srcConst = getInput().getDefiningOp<cir::ConstantOp>()) {
- if (getKind() == cir::UnaryOpKind::Plus ||
- (mlir::isa<cir::BoolType>(srcConst.getType()) &&
- getKind() == cir::UnaryOpKind::Minus))
+ if (mlir::isa<cir::BoolType>(srcConst.getType()) &&
+ getKind() == cir::UnaryOpKind::Minus)
return srcConst.getResult();
}
@@ -2733,8 +2731,6 @@ OpFoldResult cir::UnaryOp::fold(FoldAdaptor adaptor) {
val.flipAllBits();
return cir::IntAttr::get(getType(), val);
}
- case cir::UnaryOpKind::Plus:
- return attrT;
case cir::UnaryOpKind::Minus: {
APInt val = attrT.getValue();
val.negate();
@@ -2746,8 +2742,6 @@ OpFoldResult cir::UnaryOp::fold(FoldAdaptor adaptor) {
})
.Case<cir::FPAttr>([&](cir::FPAttr attrT) {
switch (getKind()) {
- case cir::UnaryOpKind::Plus:
- return attrT;
case cir::UnaryOpKind::Minus: {
APFloat val = attrT.getValue();
val.changeSign();
@@ -2761,7 +2755,6 @@ OpFoldResult cir::UnaryOp::fold(FoldAdaptor adaptor) {
switch (getKind()) {
case cir::UnaryOpKind::Not:
return cir::BoolAttr::get(getContext(), !attrT.getValue());
- case cir::UnaryOpKind::Plus:
case cir::UnaryOpKind::Minus:
return attrT;
default:
diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
index 232d320d71f37..ad6c49ca87183 100644
--- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
@@ -822,7 +822,6 @@ void LoweringPreparePass::lowerUnaryOp(cir::UnaryOp op) {
resultImag = operandImag;
break;
- case cir::UnaryOpKind::Plus:
case cir::UnaryOpKind::Minus:
resultReal = builder.createUnaryOp(loc, opKind, operandReal);
resultImag = builder.createUnaryOp(loc, opKind, operandImag);
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 8bcd040382ab3..58e7c8a65a1cd 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -2726,7 +2726,7 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
mlir::Type llvmType = getTypeConverter()->convertType(type);
mlir::Location loc = op.getLoc();
- // Integer unary operations: + - ~ ++ --
+ // Integer unary operations: - ~ ++ --
if (mlir::isa<cir::IntType>(elementType)) {
mlir::LLVM::IntegerOverflowFlags maybeNSW =
op.getNoSignedWrap() ? mlir::LLVM::IntegerOverflowFlags::nsw
@@ -2746,9 +2746,6 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
one, maybeNSW);
return mlir::success();
}
- case cir::UnaryOpKind::Plus:
- rewriter.replaceOp(op, adaptor.getInput());
- return mlir::success();
case cir::UnaryOpKind::Minus: {
mlir::Value zero;
if (isVector)
@@ -2780,7 +2777,7 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
llvm_unreachable("Unexpected unary op for int");
}
- // Floating point unary operations: + - ++ --
+ // Floating point unary operations: - ++ --
if (mlir::isa<cir::FPTypeInterface>(elementType)) {
switch (op.getKind()) {
case cir::UnaryOpKind::Inc: {
@@ -2799,9 +2796,6 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
adaptor.getInput());
return mlir::success();
}
- case cir::UnaryOpKind::Plus:
- rewriter.replaceOp(op, adaptor.getInput());
- return mlir::success();
case cir::UnaryOpKind::Minus:
rewriter.replaceOpWithNewOp<mlir::LLVM::FNegOp>(op, llvmType,
adaptor.getInput());
@@ -2818,7 +2812,6 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
switch (op.getKind()) {
case cir::UnaryOpKind::Inc:
case cir::UnaryOpKind::Dec:
- case cir::UnaryOpKind::Plus:
case cir::UnaryOpKind::Minus:
// Some of these are allowed in source code, but we shouldn't get here
// with a boolean type.
@@ -2834,19 +2827,6 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
llvm_unreachable("Unexpected unary op for bool");
}
- // Pointer unary operations: + only. (++ and -- of pointers are implemented
- // with cir.ptr_stride, not cir.unary.)
- if (mlir::isa<cir::PointerType>(elementType)) {
- switch (op.getKind()) {
- case cir::UnaryOpKind::Plus:
- rewriter.replaceOp(op, adaptor.getInput());
- return mlir::success();
- default:
- op.emitError() << "Unknown pointer unary operation during CIR lowering";
- return mlir::failure();
- }
- }
-
return op.emitError() << "Unary operation has unsupported type: "
<< elementType;
}
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index 54f18064c8cbc..92129a041fd8e 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -39,9 +39,9 @@ static const ComplexType *getComplexType(QualType type) {
}
}
-namespace {
+namespace {
class ComplexExprEmitter
- : public StmtVisitor<ComplexExprEmitter, ComplexPairTy> {
+ : public StmtVisitor<ComplexExprEmitter, ComplexPairTy> {
CodeGenFunction &CGF;
CGBuilderTy &Builder;
bool IgnoreReal;
@@ -108,7 +108,9 @@ class ComplexExprEmitter
Result->getAggregateElement(1U));
return Visit(E->getSubExpr());
}
- ComplexPairTy VisitParenExpr(ParenExpr *PE) { return Visit(PE->getSubExpr());}
+ ComplexPairTy VisitParenExpr(ParenExpr *PE) {
+ return Visit(PE->getSubExpr());
+ }
ComplexPairTy VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
return Visit(GE->getResultExpr());
}
@@ -185,15 +187,15 @@ class ComplexExprEmitter
if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
if (E->changesVolatileQualification())
- return EmitLoadOfLValue(E);
+ return EmitLoadOfLValue(E);
return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
}
ComplexPairTy VisitCallExpr(const CallExpr *E);
ComplexPairTy VisitStmtExpr(const StmtExpr *E);
// Operators.
- ComplexPairTy VisitPrePostIncDec(const UnaryOperator *E,
- bool isInc, bool isPre) {
+ ComplexPairTy VisitPrePostIncDec(const UnaryOperator *E, bool isInc,
+ bool isPre) {
LValue LV = CGF.EmitLValue(E->getSubExpr());
return CGF.EmitComplexPrePostIncDec(E, LV, isInc, isPre);
}
@@ -217,7 +219,7 @@ class ComplexExprEmitter
ComplexPairTy VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType = QualType());
ComplexPairTy VisitMinus(const UnaryOperator *E, QualType PromotionType);
- ComplexPairTy VisitUnaryNot (const UnaryOperator *E);
+ ComplexPairTy VisitUnaryNot(const UnaryOperator *E);
// LNot,Real,Imag never return complex.
ComplexPairTy VisitUnaryExtension(const UnaryOperator *E) {
return Visit(E->getSubExpr());
@@ -247,15 +249,14 @@ class ComplexExprEmitter
ComplexPairTy VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
assert(E->getType()->isAnyComplexType() && "Expected complex type!");
QualType Elem = E->getType()->castAs<ComplexType>()->getElementType();
- llvm::Constant *Null =
- llvm::Constant::getNullValue(CGF.ConvertType(Elem));
+ llvm::Constant *Null = llvm::Constant::getNullValue(CGF.ConvertType(Elem));
return ComplexPairTy(Null, Null);
}
struct BinOpInfo {
ComplexPairTy LHS;
ComplexPairTy RHS;
- QualType Ty; // Computation Type.
+ QualType Ty; // Computation Type.
FPOptions FPFeatures;
};
@@ -263,13 +264,13 @@ class ComplexExprEmitter
QualType PromotionTy = QualType());
ComplexPairTy EmitPromoted(const Expr *E, QualType PromotionTy);
ComplexPairTy EmitPromotedComplexOperand(const Expr *E, QualType PromotionTy);
- LValue EmitCompoundAssignLValue(const CompoundAssignOperator *E,
- ComplexPairTy (ComplexExprEmitter::*Func)
- (const BinOpInfo &),
- RValue &Val);
- ComplexPairTy EmitCompoundAssign(const CompoundAssignOperator *E,
- ComplexPairTy (ComplexExprEmitter::*Func)
- (const BinOpInfo &));
+ LValue EmitCompoundAssignLValue(
+ const CompoundAssignOperator *E,
+ ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo &),
+ RValue &Val);
+ ComplexPairTy EmitCompoundAssign(
+ const CompoundAssignOperator *E,
+ ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo &));
ComplexPairTy EmitBinAdd(const BinOpInfo &Op);
ComplexPairTy EmitBinSub(const BinOpInfo &Op);
@@ -381,11 +382,9 @@ class ComplexExprEmitter
// No comparisons produce a complex result.
- LValue EmitBinAssignLValue(const BinaryOperator *E,
- ComplexPairTy &Val);
- ComplexPairTy VisitBinAssign (const BinaryOperator *E);
- ComplexPairTy VisitBinComma (const BinaryOperator *E);
-
+ LValue EmitBinAssignLValue(const BinaryOperator *E, ComplexPairTy &Val);
+ ComplexPairTy VisitBinAssign(const BinaryOperator *E);
+ ComplexPairTy VisitBinComma(const BinaryOperator *E);
ComplexPairTy
VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
@@ -407,7 +406,7 @@ class ComplexExprEmitter
return Visit(E->getSelectedExpr());
}
};
-} // end anonymous namespace.
+} // end anonymous namespace.
//===----------------------------------------------------------------------===//
// Utilities
@@ -469,8 +468,6 @@ void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue,
CGF.addInstToCurrentSourceAtom(I, Val.second);
}
-
-
//===----------------------------------------------------------------------===//
// Visitor Methods
//===----------------------------------------------------------------------===//
@@ -478,18 +475,17 @@ void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue,
ComplexPairTy ComplexExprEmitter::VisitExpr(Expr *E) {
CGF.ErrorUnsupported(E, "complex expression");
llvm::Type *EltTy =
- CGF.ConvertType(getComplexType(E->getType())->getElementType());
+ CGF.ConvertType(getComplexType(E->getType())->getElementType());
llvm::Value *U = llvm::PoisonValue::get(EltTy);
return ComplexPairTy(U, U);
}
-ComplexPairTy ComplexExprEmitter::
-VisitImaginaryLiteral(const ImaginaryLiteral *IL) {
+ComplexPairTy
+ComplexExprEmitter::VisitImaginaryLiteral(const ImaginaryLiteral *IL) {
llvm::Value *Imag = CGF.EmitScalarExpr(IL->getSubExpr());
return ComplexPairTy(llvm::Constant::getNullValue(Imag->getType()), Imag);
}
-
ComplexPairTy ComplexExprEmitter::VisitCallExpr(const CallExpr *E) {
if (E->getCallReturnType(CGF.getContext())->isReferenceType())
return EmitLoadOfLValue(E);
@@ -539,7 +535,8 @@ ComplexPairTy ComplexExprEmitter::EmitScalarToComplexCast(llvm::Value *Val,
ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
QualType DestTy) {
switch (CK) {
- case CK_Dependent: llvm_unreachable("dependent cast kind in IR gen!");
+ case CK_Dependent:
+ llvm_unreachable("dependent cast kind in IR gen!");
// Atomic to non-atomic casts may be more than a no-op for some platforms and
// for some types.
@@ -691,10 +688,10 @@ ComplexPairTy ComplexExprEmitter::VisitMinus(const UnaryOperator *E,
llvm::Value *ResR, *ResI;
if (Op.first->getType()->isFloatingPointTy()) {
- ResR = Builder.CreateFNeg(Op.first, "neg.r");
+ ResR = Builder.CreateFNeg(Op.first, "neg.r");
ResI = Builder.CreateFNeg(Op.second, "neg.i");
} else {
- ResR = Builder.CreateNeg(Op.first, "neg.r");
+ ResR = Builder.CreateNeg(Op.first, "neg.r");
ResI = Builder.CreateNeg(Op.second, "neg.i");
}
return ComplexPairTy(ResR, ResI);
@@ -719,14 +716,14 @@ ComplexPairTy ComplexExprEmitter::EmitBinAdd(const BinOpInfo &Op) {
if (Op.LHS.first->getType()->isFloatingPointTy()) {
CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
- ResR = Builder.CreateFAdd(Op.LHS.first, Op.RHS.first, "add.r");
+ ResR = Builder.CreateFAdd(Op.LHS.first, Op.RHS.first, "add.r");
if (Op.LHS.second && Op.RHS.second)
ResI = Builder.CreateFAdd(Op.LHS.second, Op.RHS.second, "add.i");
else
ResI = Op.LHS.second ? Op.LHS.second : Op.RHS.second;
assert(ResI && "Only one operand may be real!");
} else {
- ResR = Builder.CreateAdd(Op.LHS.first, Op.RHS.first, "add.r");
+ ResR = Builder.CreateAdd(Op.LHS.first, Op.RHS.first, "add.r");
assert(Op.LHS.second && Op.RHS.second &&
"Both operands of integer complex operators must be complex!");
ResI = Builder.CreateAdd(Op.LHS.second, Op.RHS.second, "add.i");
@@ -884,11 +881,13 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) {
// Finally continue execution by phi-ing together the different
// computation paths.
CGF.EmitBlock(ContBB);
- llvm::PHINode *RealPHI = Builder.CreatePHI(ResR->getType(), 3, "real_mul_phi");
+ llvm::PHINode *RealPHI =
+ Builder.CreatePHI(ResR->getType(), 3, "real_mul_phi");
RealPHI->addIncoming(ResR, OrigBB);
RealPHI->addIncoming(ResR, INaNBB);
RealPHI->addIncoming(LibCallR, LibCallBB);
- llvm::PHINode *ImagPHI = Builder.CreatePHI(ResI->getType(), 3, "imag_mul_phi");
+ llvm::PHINode *ImagPHI =
+ Builder.CreatePHI(ResI->getType(), 3, "imag_mul_phi");
ImagPHI->addIncoming(ResI, OrigBB);
ImagPHI->addIncoming(ResI, INaNBB);
ImagPHI->addIncoming(LibCallI, LibCallBB);
@@ -1097,7 +1096,9 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) {
llvm::Value *Tmp8 = Builder.CreateMul(LHSr, RHSi); // a*d
llvm::Value *Tmp9 = Builder.CreateSub(Tmp7, Tmp8); // bc-ad
- if (Op.Ty->castAs<ComplexType>()->getElementType()->isUnsignedIntegerType()) {
+ if (Op.Ty->castAs<ComplexType>()
+ ->getElementType()
+ ->isUnsignedIntegerType()) {
DSTr = Builder.CreateUDiv(Tmp3, Tmp6);
DSTi = Builder.CreateUDiv(Tmp9, Tmp6);
} else {
@@ -1209,11 +1210,9 @@ ComplexExprEmitter::EmitBinOps(const BinaryOperator *E,
return Ops;
}
-
-LValue ComplexExprEmitter::
-EmitCompoundAssignLValue(const CompoundAssignOperator *E,
- ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&),
- RValue &Val) {
+LValue ComplexExprEmitter::EmitCompoundAssignLValue(
+ const CompoundAssignOperator *E,
+ ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo &), RValue &Val) {
TestAndClearIgnoreReal();
TestAndClearIgnoreImag();
QualType LHSTy = E->getLHS()->getType();
@@ -1323,9 +1322,9 @@ EmitCompoundAssignLValue(const CompoundAssignOperator *E,
}
// Compound assignments.
-ComplexPairTy ComplexExprEmitter::
-EmitCompoundAssign(const CompoundAssignOperator *E,
- ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&)){
+ComplexPairTy ComplexExprEmitter::EmitCompoundAssign(
+ const CompoundAssignOperator *E,
+ ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo &)) {
RValue Val;
LValue LV = EmitCompoundAssignLValue(E, Func, Val);
@@ -1381,8 +1380,8 @@ ComplexPairTy ComplexExprEmitter::VisitBinComma(const BinaryOperator *E) {
return Visit(E->getRHS());
}
-ComplexPairTy ComplexExprEmitter::
-VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
+ComplexPairTy ComplexExprEmitter::VisitAbstractConditionalOperator(
+ const AbstractConditionalOperator *E) {
TestAndClearIgnoreReal();
TestAndClearIgnoreImag();
llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
@@ -1392,7 +1391,6 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
// Bind the common expression if necessary.
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
-
CodeGenFunction::ConditionalEvaluation eval(CGF);
CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
CGF.getProfileCount(E));
@@ -1431,12 +1429,12 @@ ComplexPairTy ComplexExprEmitter::VisitChooseExpr(ChooseExpr *E) {
}
ComplexPairTy ComplexExprEmitter::VisitInitListExpr(InitListExpr *E) {
- bool Ignore = TestAndClearIgnoreReal();
- (void)Ignore;
- assert (Ignore == false && "init list ignored");
- Ignore = TestAndClearIgnoreImag();
- (void)Ignore;
- assert (Ignore == false && "init list ignored");
+ bool Ignore = TestAndClearIgnoreReal();
+ (void)Ignore;
+ assert(Ignore == false && "init list ignored");
+ Ignore = TestAndClearIgnoreImag();
+ (void)Ignore;
+ assert(Ignore == false && "init list ignored");
if (E->getNumInits() == 2) {
llvm::Value *Real = CGF.EmitScalarExpr(E->getInit(0));
@@ -1449,8 +1447,8 @@ ComplexPairTy ComplexExprEmitter::VisitInitListExpr(InitListExpr *E) {
// Empty init list initializes to null
assert(E->getNumInits() == 0 && "Unexpected number of inits");
QualType Ty = E->getType()->castAs<ComplexType>()->getElementType();
- llvm::Type* LTy = CGF.ConvertType(Ty);
- llvm::Value* zeroConstant = llvm::Constant::getNullValue(LTy);
+ llvm::Type *LTy = CGF.ConvertType(Ty);
+ llvm::Value *zeroConstant = llvm::Constant::getNullValue(LTy);
return ComplexPairTy(zeroConstant, zeroConstant);
}
@@ -1461,7 +1459,7 @@ ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) {
if (!ArgValue.isValid()) {
CGF.ErrorUnsupported(E, "complex va_arg expression");
llvm::Type *EltTy =
- CGF.ConvertType(E->getType()->castAs<ComplexType>()->getElementType());
+ CGF.ConvertType(E->getType()->castAs<ComplexType>()->getElementType());
llvm::Value *U = llvm::PoisonValue::get(EltTy);
return ComplexPairTy(U, U);
}
@@ -1489,7 +1487,7 @@ void CodeGenFunction::EmitComplexExprIntoLValue(const Expr *E, LValue dest,
assert(E && getComplexType(E->getType()) &&
"Invalid complex expression to emit");
ComplexExprEmitter Emitter(*this);
- ComplexPairTy Val = Emitter.Visit(const_cast<Expr*>(E));
+ ComplexPairTy Val = Emitter.Visit(const_cast<Expr *>(E));
Emitter.EmitStoreOfComplex(Val, dest, isInit);
}
@@ -1520,26 +1518,29 @@ typedef ComplexPairTy (ComplexExprEmitter::*CompoundFunc)(
static CompoundFunc getComplexOp(BinaryOperatorKind Op) {
switch (Op) {
- case BO_MulAssign: return &ComplexExprEmitter::EmitBinMul;
- case BO_DivAssign: return &ComplexExprEmitter::EmitBinDiv;
- case BO_SubAssign: return &ComplexExprEmitter::EmitBinSub;
- case BO_AddAssign: return &ComplexExprEmitter::EmitBinAdd;
+ case BO_MulAssign:
+ return &ComplexExprEmitter::EmitBinMul;
+ case BO_DivAssign:
+ return &ComplexExprEmitter::EmitBinDiv;
+ case BO_SubAssign:
+ return &ComplexExprEmitter::EmitBinSub;
+ case BO_AddAssign:
+ return &ComplexExprEmitter::EmitBinAdd;
default:
llvm_unreachable("unexpected complex compound assignment");
}
}
-LValue CodeGenFunction::
-EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E) {
+LValue CodeGenFunction::EmitComplexCompoundAssignmentLValue(
+ const CompoundAssignOperator *E) {
ApplyAtomGroup Grp(getDebugInfo());
CompoundFunc Op = getComplexOp(E->getOpcode());
RValue Val;
return ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);
}
-LValue CodeGenFunction::
-EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
- llvm::Value *&Result) {
+LValue CodeGenFunction::EmitScalarCompoundAssignWithComplex(
+ const CompoundAssignOperator *E, llvm::Value *&Result) {
// Key Instructions: Don't need to create an atom group here; one will already
// be active through scalar handling code.
CompoundFunc Op = getComplexOp(E->getOpcode());
diff --git a/clang/test/CIR/CodeGen/complex-unary.cpp b/clang/test/CIR/CodeGen/complex-unary.cpp
index a8e434b903763..31cc0d8b8de56 100644
--- a/clang/test/CIR/CodeGen/complex-unary.cpp
+++ b/clang/test/CIR/CodeGen/complex-unary.cpp
@@ -293,27 +293,19 @@ void foo7() {
// CIR-BEFORE: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
// CIR-BEFORE: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b", init]
// CIR-BEFORE: %[[TMP:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
-// CIR-BEFORE: %[[COMPLEX_PLUS:.*]] = cir.unary(plus, %[[TMP]]) : !cir.complex<!cir.float>, !cir.complex<!cir.float>
-// CIR-BEFORE: cir.store{{.*}} %[[COMPLEX_PLUS]], %[[B_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+// CIR-BEFORE-NOT: cir.unary
+// CIR-BEFORE: cir.store{{.*}} %[[TMP]], %[[B_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
// CIR-AFTER: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
// CIR-AFTER: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b", init]
// CIR-AFTER: %[[TMP:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
-// CIR-AFTER: %[[REAL:.*]] = cir.complex.real %[[TMP]] : !cir.complex<!cir.float> -> !cir.float
-// CIR-AFTER: %[[IMAG:.*]] = cir.complex.imag %[[TMP]] : !cir.complex<!cir.float> -> !cir.float
-// CIR-AFTER: %[[REAL_PLUS:.*]] = cir.unary(plus, %[[REAL]]) : !cir.float, !cir.float
-// CIR-AFTER: %[[IMAG_PLUS:.*]] = cir.unary(plus, %[[IMAG]]) : !cir.float, !cir.float
-// CIR-AFTER: %[[NEW_COMPLEX:.*]] = cir.complex.create %[[REAL_PLUS]], %[[IMAG_PLUS]] : !cir.float -> !cir.complex<!cir.float>
-// CIR-AFTER: cir.store{{.*}} %[[NEW_COMPLEX]], %[[B_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+// CIR-AFTER-NOT: cir.unary
+// CIR-AFTER: cir.store{{.*}} %[[TMP]], %[[B_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
// LLVM: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
// LLVM: %[[TMP:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
-// LLVM: %[[REAL:.*]] = extractvalue { float, float } %[[TMP]], 0
-// LLVM: %[[IMAG:.*]] = extractvalue { float, float } %[[TMP]], 1
-// LLVM: %[[RESULT_TMP:.*]] = insertvalue { float, float } {{.*}}, float %[[REAL]], 0
-// LLVM: %[[RESULT_VAL:.*]] = insertvalue { float, float } %[[RESULT_TMP]], float %[[IMAG]], 1
-// LLVM: store { float, float } %[[RESULT_VAL]], ptr %[[B_ADDR]], align 4
+// LLVM: store { float, float } %[[TMP]], ptr %[[B_ADDR]], align 4
// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
// OGCG: %[[B_ADDR:.*]] = alloca { float, float }, align 4
@@ -381,8 +373,8 @@ void foo9() {
// CIR-BEFORE: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.f16>, !cir.ptr<!cir.complex<!cir.f16>>, ["b", init]
// CIR-BEFORE: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.f16>>, !cir.complex<!cir.f16>
// CIR-BEFORE: %[[A_COMPLEX_F32:.*]] = cir.cast float_complex %[[TMP_A]] : !cir.complex<!cir.f16> -> !cir.complex<!cir.float>
-// CIR-BEFORE: %[[RESULT:.*]] = cir.unary(plus, %[[A_COMPLEX_F32]]) : !cir.complex<!cir.float>, !cir.complex<!cir.float>
-// CIR-BEFORE: %[[A_COMPLEX_F16:.*]] = cir.cast float_complex %[[RESULT]] : !cir.complex<!cir.float> -> !cir.complex<!cir.f16>
+// CIR-BEFORE-NOT: cir.unary
+// CIR-BEFORE: %[[A_COMPLEX_F16:.*]] = cir.cast float_complex %[[A_COMPLEX_F32]] : !cir.complex<!cir.float> -> !cir.complex<!cir.f16>
// CIR-BEFORE: cir.store{{.*}} %[[A_COMPLEX_F16]], %[[B_ADDR]] : !cir.complex<!cir.f16>, !cir.ptr<!cir.complex<!cir.f16>>
// CIR-AFTER: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.f16>, !cir.ptr<!cir.complex<!cir.f16>>, ["a"]
@@ -393,13 +385,9 @@ void foo9() {
// CIR-AFTER: %[[A_REAL_F32:.*]] = cir.cast floating %[[A_REAL]] : !cir.f16 -> !cir.float
// CIR-AFTER: %[[A_IMAG_F32:.*]] = cir.cast floating %[[A_IMAG]] : !cir.f16 -> !cir.float
// CIR-AFTER: %[[A_COMPLEX_F32:.*]] = cir.complex.create %[[A_REAL_F32]], %[[A_IMAG_F32]] : !cir.float -> !cir.complex<!cir.float>
-// CIR-AFTER: %[[A_REAL_F32:.*]] = cir.complex.real %[[A_COMPLEX_F32]] : !cir.complex<!cir.float> -> !cir.float
-// CIR-AFTER: %[[A_IMAG_F32:.*]] = cir.complex.imag %[[A_COMPLEX_F32]] : !cir.complex<!cir.float> -> !cir.float
-// CIR-AFTER: %[[RESULT_REAL_F32:.*]] = cir.unary(plus, %[[A_REAL_F32]]) : !cir.float, !cir.float
-// CIR-AFTER: %[[RESULT_IMAG_F32:.*]] = cir.unary(plus, %[[A_IMAG_F32]]) : !cir.float, !cir.float
-// CIR-AFTER: %[[RESULT_COMPLEX_F32:.*]] = cir.complex.create %[[RESULT_REAL_F32]], %[[RESULT_IMAG_F32]] : !cir.float -> !cir.complex<!cir.float>
-// CIR-AFTER: %[[RESULT_REAL_F32:.*]] = cir.complex.real %[[RESULT_COMPLEX_F32]] : !cir.complex<!cir.float> -> !cir.float
-// CIR-AFTER: %[[RESULT_IMAG_F32:.*]] = cir.complex.imag %[[RESULT_COMPLEX_F32]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER: %[[RESULT_REAL_F32:.*]] = cir.complex.real %[[A_COMPLEX_F32]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER: %[[RESULT_IMAG_F32:.*]] = cir.complex.imag %[[A_COMPLEX_F32]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-NOT: cir.unary
// CIR-AFTER: %[[RESULT_REAL_F16:.*]] = cir.cast floating %[[RESULT_REAL_F32]] : !cir.float -> !cir.f16
// CIR-AFTER: %[[RESULT_IMAG_F16:.*]] = cir.cast floating %[[RESULT_IMAG_F32]] : !cir.float -> !cir.f16
// CIR-AFTER: %[[RESULT_COMPLEX_F16:.*]] = cir.complex.create %[[RESULT_REAL_F16]], %[[RESULT_IMAG_F16]] : !cir.f16 -> !cir.complex<!cir.f16>
@@ -412,10 +400,6 @@ void foo9() {
// LLVM: %[[A_IMAG:.*]] = extractvalue { half, half } %[[TMP_A]], 1
// LLVM: %[[A_REAL_F32:.*]] = fpext half %[[A_REAL]] to float
// LLVM: %[[A_IMAG_F32:.*]] = fpext half %[[A_IMAG]] to float
-// LLVM: %[[TMP_A_COMPLEX_F32:.*]] = insertvalue { float, float } {{.*}}, float %[[A_REAL_F32]], 0
-// LLVM: %[[A_COMPLEX_F32:.*]] = insertvalue { float, float } %[[TMP_A_COMPLEX_F32]], float %[[A_IMAG_F32]], 1
-// LLVM: %[[TMP_A_COMPLEX_F32:.*]] = insertvalue { float, float } {{.*}}, float %[[A_REAL_F32]], 0
-// LLVM: %[[A_COMPLEX_F32:.*]] = insertvalue { float, float } %[[TMP_A_COMPLEX_F32]], float %[[A_IMAG_F32]], 1
// LLVM: %[[A_REAL_F16:.*]] = fptrunc float %[[A_REAL_F32]] to half
// LLVM: %[[A_IMAG_F16:.*]] = fptrunc float %[[A_IMAG_F32]] to half
// LLVM: %[[TMP_RESULT_COMPLEX_F16:.*]] = insertvalue { half, half } {{.*}}, half %[[A_REAL_F16]], 0
diff --git a/clang/test/CIR/CodeGen/complex.cpp b/clang/test/CIR/CodeGen/complex.cpp
index 48b7dd72ae661..d54fcb1c3ccf4 100644
--- a/clang/test/CIR/CodeGen/complex.cpp
+++ b/clang/test/CIR/CodeGen/complex.cpp
@@ -1034,21 +1034,13 @@ void real_on_non_glvalue() {
// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b", init]
// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
-// CIR: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
-// CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
-// CIR: %[[A_REAL_PLUS:.*]] = cir.unary(plus, %[[A_REAL]]) : !cir.float, !cir.float
-// CIR: %[[A_IMAG_PLUS:.*]] = cir.unary(plus, %[[A_IMAG]]) : !cir.float, !cir.float
-// CIR: %[[RESULT:.*]] = cir.complex.create %[[A_REAL_PLUS]], %[[A_IMAG_PLUS]] : !cir.float -> !cir.complex<!cir.float>
-// CIR: %[[RESULT_REAL:.*]] = cir.complex.real %[[RESULT]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[RESULT_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
// CIR: cir.store{{.*}} %[[RESULT_REAL]], %[[B_ADDR]] : !cir.float, !cir.ptr<!cir.float>
// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
// LLVM: %[[B_ADDR:.*]] = alloca float, i64 1, align 4
// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
// LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
-// LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
-// LLVM: %[[TMP_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[A_REAL]], 0
-// LLVM: %[[RESULT:.*]] = insertvalue { float, float } %[[TMP_RESULT]], float %[[A_IMAG]], 1
// LLVM: store float %[[A_REAL]], ptr %[[B_ADDR]], align 4
// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
@@ -1067,21 +1059,13 @@ void imag_on_non_glvalue() {
// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b", init]
// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
-// CIR: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
-// CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
-// CIR: %[[A_REAL_PLUS:.*]] = cir.unary(plus, %[[A_REAL]]) : !cir.float, !cir.float
-// CIR: %[[A_IMAG_PLUS:.*]] = cir.unary(plus, %[[A_IMAG]]) : !cir.float, !cir.float
-// CIR: %[[RESULT:.*]] = cir.complex.create %[[A_REAL_PLUS]], %[[A_IMAG_PLUS]] : !cir.float -> !cir.complex<!cir.float>
-// CIR: %[[RESULT_IMAG:.*]] = cir.complex.imag %[[RESULT]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[RESULT_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
// CIR: cir.store{{.*}} %[[RESULT_IMAG]], %[[B_ADDR]] : !cir.float, !cir.ptr<!cir.float>
// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
// LLVM: %[[B_ADDR:.*]] = alloca float, i64 1, align 4
// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
-// LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
// LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
-// LLVM: %[[TMP_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[A_REAL]], 0
-// LLVM: %[[RESULT:.*]] = insertvalue { float, float } %[[TMP_RESULT]], float %[[A_IMAG]], 1
// LLVM: store float %[[A_IMAG]], ptr %[[B_ADDR]], align 4
// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
diff --git a/clang/test/CIR/CodeGen/coro-task.cpp b/clang/test/CIR/CodeGen/coro-task.cpp
index 637b058443bc7..b52f0f1871079 100644
--- a/clang/test/CIR/CodeGen/coro-task.cpp
+++ b/clang/test/CIR/CodeGen/coro-task.cpp
@@ -569,8 +569,7 @@ folly::coro::Task<int> go4() {
// Get the lambda invoker ptr via `lambda operator folly::coro::Task<int> (*)(int const&)()`
// CIR: %[[INVOKER:.*]] = cir.call @_ZZ3go4vENK3$_0cvPFN5folly4coro4TaskIiEERKiEEv(%{{.*}}) nothrow : {{.*}} -> (!cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>> {llvm.noundef})
-// CIR: %[[PLUS:.*]] = cir.unary(plus, %[[INVOKER]]) : !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>>, !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>>
-// CIR: cir.store{{.*}} %[[PLUS]], %[[FN_ADDR:.*]] : !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>>, !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>>>
+// CIR: cir.store{{.*}} %[[INVOKER]], %[[FN_ADDR:.*]] : !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>>, !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>>>
// CIR: cir.scope {
// CIR: %[[ARG:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["ref.tmp2", init] {alignment = 4 : i64}
// CIR: %[[FN:.*]] = cir.load{{.*}} %[[FN_ADDR]] : !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>>>, !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> ![[IntTask]]>>
diff --git a/clang/test/CIR/CodeGen/lambda-static-invoker.cpp b/clang/test/CIR/CodeGen/lambda-static-invoker.cpp
index 2b00feccbc79a..25c17d6b37dde 100644
--- a/clang/test/CIR/CodeGen/lambda-static-invoker.cpp
+++ b/clang/test/CIR/CodeGen/lambda-static-invoker.cpp
@@ -121,10 +121,9 @@ int g3() {
// 1. Use `operator int (*)(int const&)()` to retrieve the fnptr to `__invoke()`.
// CIR: %[[OPERATOR_RESULT:.*]] = cir.call @_ZZ2g3vENK3$_0cvPFiRKiEEv(%[[LAM_ALLOCA]]){{.*}}
-// CIR: %[[PLUS:.*]] = cir.unary(plus, %[[OPERATOR_RESULT]])
// 2. Load ptr to `__invoke()`.
-// CIR: cir.store{{.*}} %[[PLUS]], %[[FN_ADDR]]
+// CIR: cir.store{{.*}} %[[OPERATOR_RESULT]], %[[FN_ADDR]]
// CIR: %[[FN:.*]] = cir.load{{.*}} %[[FN_ADDR]]
// CIR: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
// CIR: cir.store{{.*}} %[[THREE]], %[[REF_TMP1]]
diff --git a/clang/test/CIR/CodeGen/unary.cpp b/clang/test/CIR/CodeGen/unary.cpp
index cc6f2ade38f3f..7d4c8f27e4545 100644
--- a/clang/test/CIR/CodeGen/unary.cpp
+++ b/clang/test/CIR/CodeGen/unary.cpp
@@ -13,7 +13,7 @@ unsigned up0() {
// CHECK: cir.func{{.*}} @_Z3up0v() -> (!u32i{{.*}})
// CHECK: %[[A:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init]
// CHECK: %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
-// CHECK: %[[OUTPUT:.*]] = cir.unary(plus, %[[INPUT]])
+// CHECK-NOT: cir.unary
// LLVM: define{{.*}} i32 @_Z3up0v()
// LLVM: %[[RV:.*]] = alloca i32, i64 1, align 4
@@ -231,7 +231,7 @@ float fpPlus() {
// CHECK: cir.func{{.*}} @_Z6fpPlusv() -> (!cir.float{{.*}})
// CHECK: %[[A:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
// CHECK: %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
-// CHECK: %[[OUTPUT:.*]] = cir.unary(plus, %[[INPUT]])
+// CHECK-NOT: cir.unary
// LLVM: define{{.*}} float @_Z6fpPlusv()
// LLVM: %[[RV:.*]] = alloca float, i64 1, align 4
@@ -411,7 +411,7 @@ void chars(char c) {
int c1 = +c;
// CHECK: %[[PROMO:.*]] = cir.cast integral %{{.+}} : !s8i -> !s32i
- // CHECK: cir.unary(plus, %[[PROMO]]) : !s32i, !s32i
+ // CHECK-NOT: cir.unary
int c2 = -c;
// CHECK: %[[PROMO:.*]] = cir.cast integral %{{.+}} : !s8i -> !s32i
// CHECK: cir.unary(minus, %[[PROMO]]) nsw : !s32i, !s32i
@@ -432,8 +432,8 @@ _Float16 fp16UPlus(_Float16 f) {
// CHECK: cir.func{{.*}} @_Z9fp16UPlusDF16_({{.*}}) -> (!cir.f16{{.*}})
// CHECK: %[[INPUT:.*]] = cir.load{{.*}} %[[F:.*]]
// CHECK: %[[PROMOTED:.*]] = cir.cast floating %[[INPUT]] : !cir.f16 -> !cir.float
-// CHECK: %[[RESULT:.*]] = cir.unary(plus, %[[PROMOTED]])
-// CHECK: %[[UNPROMOTED:.*]] = cir.cast floating %[[RESULT]] : !cir.float -> !cir.f16
+// CHECK-NOT: cir.unary
+// CHECK: %[[UNPROMOTED:.*]] = cir.cast floating %[[PROMOTED]] : !cir.float -> !cir.f16
// LLVM: define{{.*}} half @_Z9fp16UPlusDF16_({{.*}})
// LLVM: %[[F_LOAD:.*]] = load half, ptr %{{.*}}, align 2
@@ -567,9 +567,8 @@ void f16NestedUPlus() {
// CHECK: %[[B_ADDR:.*]] = cir.alloca !cir.f16, !cir.ptr<!cir.f16>, ["b", init]
// CHECK: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.f16>, !cir.f16
// CHECK: %[[A_F32:.*]] = cir.cast floating %[[TMP_A]] : !cir.f16 -> !cir.float
-// CHECK: %[[A_PLUS:.*]] = cir.unary(plus, %[[A_F32]]) : !cir.float, !cir.float
-// CHECK: %[[RESULT_F32:.*]] = cir.unary(plus, %[[A_PLUS]]) : !cir.float, !cir.float
-// CHECK: %[[RESULT:.*]] = cir.cast floating %[[RESULT_F32]] : !cir.float -> !cir.f16
+// CHECK-NOT: cir.unary
+// CHECK: %[[RESULT:.*]] = cir.cast floating %[[A_F32]] : !cir.float -> !cir.f16
// CHECK: cir.store{{.*}} %[[RESULT]], %[[B_ADDR]] : !cir.f16, !cir.ptr<!cir.f16>
// LLVM: define{{.*}} void @_Z14f16NestedUPlusv()
diff --git a/clang/test/CIR/CodeGen/vector-ext.cpp b/clang/test/CIR/CodeGen/vector-ext.cpp
index e7d6b4974c6ae..a32bc94bae103 100644
--- a/clang/test/CIR/CodeGen/vector-ext.cpp
+++ b/clang/test/CIR/CodeGen/vector-ext.cpp
@@ -332,8 +332,7 @@ void foo8() {
// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
// CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
// CIR: %[[TMP1:.*]] = cir.load{{.*}} %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
-// CIR: %[[PLUS:.*]] = cir.unary(plus, %[[TMP1]]) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
-// CIR: cir.store{{.*}} %[[PLUS]], %[[PLUS_RES]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+// CIR: cir.store{{.*}} %[[TMP1]], %[[PLUS_RES]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
// CIR: %[[TMP2:.*]] = cir.load{{.*}} %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
// CIR: %[[MINUS:.*]] = cir.unary(minus, %[[TMP2]]) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
// CIR: cir.store{{.*}} %[[MINUS]], %[[MINUS_RES]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
diff --git a/clang/test/CIR/CodeGen/vector.cpp b/clang/test/CIR/CodeGen/vector.cpp
index 8a804ad6fa2f1..26b4e4a8f7f8a 100644
--- a/clang/test/CIR/CodeGen/vector.cpp
+++ b/clang/test/CIR/CodeGen/vector.cpp
@@ -319,8 +319,7 @@ void foo8() {
// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
// CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
// CIR: %[[TMP1:.*]] = cir.load{{.*}} %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
-// CIR: %[[PLUS:.*]] = cir.unary(plus, %[[TMP1]]) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
-// CIR: cir.store{{.*}} %[[PLUS]], %[[PLUS_RES]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+// CIR: cir.store{{.*}} %[[TMP1]], %[[PLUS_RES]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
// CIR: %[[TMP2:.*]] = cir.load{{.*}} %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
// CIR: %[[MINUS:.*]] = cir.unary(minus, %[[TMP2]]) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
// CIR: cir.store{{.*}} %[[MINUS]], %[[MINUS_RES]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
diff --git a/clang/test/CIR/IR/unary.cir b/clang/test/CIR/IR/unary.cir
index d01d4eb3c920a..0fdf1dcfdfb47 100644
--- a/clang/test/CIR/IR/unary.cir
+++ b/clang/test/CIR/IR/unary.cir
@@ -9,42 +9,38 @@ module {
cir.func @test_unary_unsigned() {
%0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["a"] {alignment = 4 : i64}
%1 = cir.load %0 : !cir.ptr<!u32i>, !u32i
- %2 = cir.unary(plus, %1) : !u32i, !u32i
- %3 = cir.unary(minus, %1) : !u32i, !u32i
- %4 = cir.unary(not, %1) : !u32i, !u32i
- %5 = cir.unary(inc, %1) : !u32i, !u32i
- %6 = cir.unary(dec, %1) : !u32i, !u32i
+ %2 = cir.unary(minus, %1) : !u32i, !u32i
+ %3 = cir.unary(not, %1) : !u32i, !u32i
+ %4 = cir.unary(inc, %1) : !u32i, !u32i
+ %5 = cir.unary(dec, %1) : !u32i, !u32i
cir.return
}
// CHECK: cir.func{{.*}} @test_unary_unsigned() {
// CHECK: %0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["a"] {alignment = 4 : i64}
// CHECK: %1 = cir.load %0 : !cir.ptr<!u32i>, !u32i
-// CHECK: %2 = cir.unary(plus, %1) : !u32i, !u32i
-// CHECK: %3 = cir.unary(minus, %1) : !u32i, !u32i
-// CHECK: %4 = cir.unary(not, %1) : !u32i, !u32i
-// CHECK: %5 = cir.unary(inc, %1) : !u32i, !u32i
-// CHECK: %6 = cir.unary(dec, %1) : !u32i, !u32i
+// CHECK: %2 = cir.unary(minus, %1) : !u32i, !u32i
+// CHECK: %3 = cir.unary(not, %1) : !u32i, !u32i
+// CHECK: %4 = cir.unary(inc, %1) : !u32i, !u32i
+// CHECK: %5 = cir.unary(dec, %1) : !u32i, !u32i
// CHECK: cir.return
// CHECK: }
cir.func @test_unary_signed() {
%0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["a"] {alignment = 4 : i64}
%1 = cir.load %0 : !cir.ptr<!s32i>, !s32i
- %2 = cir.unary(plus, %1) : !s32i, !s32i
- %3 = cir.unary(minus, %1) nsw : !s32i, !s32i
- %4 = cir.unary(not, %1) : !s32i, !s32i
- %5 = cir.unary(inc, %1) nsw : !s32i, !s32i
- %6 = cir.unary(dec, %1) nsw : !s32i, !s32i
+ %2 = cir.unary(minus, %1) nsw : !s32i, !s32i
+ %3 = cir.unary(not, %1) : !s32i, !s32i
+ %4 = cir.unary(inc, %1) nsw : !s32i, !s32i
+ %5 = cir.unary(dec, %1) nsw : !s32i, !s32i
cir.return
}
// CHECK: cir.func{{.*}} @test_unary_signed() {
// CHECK: %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["a"] {alignment = 4 : i64}
// CHECK: %1 = cir.load %0 : !cir.ptr<!s32i>, !s32i
-// CHECK: %2 = cir.unary(plus, %1) : !s32i, !s32i
-// CHECK: %3 = cir.unary(minus, %1) nsw : !s32i, !s32i
-// CHECK: %4 = cir.unary(not, %1) : !s32i, !s32i
-// CHECK: %5 = cir.unary(inc, %1) nsw : !s32i, !s32i
-// CHECK: %6 = cir.unary(dec, %1) nsw : !s32i, !s32i
+// CHECK: %2 = cir.unary(minus, %1) nsw : !s32i, !s32i
+// CHECK: %3 = cir.unary(not, %1) : !s32i, !s32i
+// CHECK: %4 = cir.unary(inc, %1) nsw : !s32i, !s32i
+// CHECK: %5 = cir.unary(dec, %1) nsw : !s32i, !s32i
// CHECK: cir.return
// CHECK: }
}
diff --git a/clang/test/CIR/Transforms/canonicalize.cir b/clang/test/CIR/Transforms/canonicalize.cir
index cfac73ecdb738..7476126818082 100644
--- a/clang/test/CIR/Transforms/canonicalize.cir
+++ b/clang/test/CIR/Transforms/canonicalize.cir
@@ -117,78 +117,6 @@ module {
// CHECK-NEXT: %[[CONST:.*]] = cir.const #cir.int<4294967294> : !u32i
// CHECK-NEXT: cir.return %[[CONST]] : !u32i
- cir.func @unary_plus_true() -> !cir.bool {
- %0 = cir.const #true
- %1 = cir.unary(plus, %0) : !cir.bool, !cir.bool
- cir.return %1 : !cir.bool
- }
- // CHECK: cir.func{{.*}} @unary_plus_true() -> !cir.bool
- // CHECK-NEXT: %[[CONST:.*]] = cir.const #true
- // CHECK-NEXT: cir.return %[[CONST]] : !cir.bool
-
- cir.func @unary_plus_false() -> !cir.bool {
- %0 = cir.const #false
- %1 = cir.unary(plus, %0) : !cir.bool, !cir.bool
- cir.return %1 : !cir.bool
- }
- // CHECK: cir.func{{.*}} @unary_plus_false() -> !cir.bool
- // CHECK-NEXT: %[[CONST:.*]] = cir.const #false
- // CHECK-NEXT: cir.return %[[CONST]] : !cir.bool
-
- cir.func @unary_plus_int() -> !s32i {
- %0 = cir.const #cir.int<1> : !s32i
- %1 = cir.unary(plus, %0) : !s32i, !s32i
- cir.return %1 : !s32i
- }
- // CHECK: cir.func{{.*}} @unary_plus_int() -> !s32i
- // CHECK-NEXT: %[[CONST:.*]] = cir.const #cir.int<1> : !s32i
- // CHECK-NEXT: cir.return %[[CONST]] : !s32i
-
- cir.func @unary_plus_uint() -> !u32i {
- %0 = cir.const #cir.int<1> : !u32i
- %1 = cir.unary(plus, %0) : !u32i, !u32i
- cir.return %1 : !u32i
- }
- // CHECK: cir.func{{.*}} @unary_plus_uint() -> !u32i
- // CHECK-NEXT: %[[CONST:.*]] = cir.const #cir.int<1> : !u32i
- // CHECK-NEXT: cir.return %[[CONST]] : !u32i
-
- cir.func @unary_plus_float() -> !cir.float {
- %0 = cir.const #cir.fp<1.100000e+00> : !cir.float
- %1 = cir.unary(plus, %0) : !cir.float, !cir.float
- cir.return %1 : !cir.float
- }
- // CHECK: cir.func{{.*}} @unary_plus_float() -> !cir.float
- // CHECK-NEXT: %[[CONST:.*]] = cir.const #cir.fp<1.100000e+00> : !cir.float
- // CHECK-NEXT: cir.return %[[CONST]] : !cir.float
-
- cir.func @unary_plus_double() -> !cir.double {
- %0 = cir.const #cir.fp<1.100000e+00> : !cir.double
- %1 = cir.unary(plus, %0) : !cir.double, !cir.double
- cir.return %1 : !cir.double
- }
- // CHECK: cir.func{{.*}} @unary_plus_double() -> !cir.double
- // CHECK-NEXT: %[[CONST:.*]] = cir.const #cir.fp<1.100000e+00> : !cir.double
- // CHECK-NEXT: cir.return %[[CONST]] : !cir.double
-
- cir.func @unary_plus_nan() -> !cir.float {
- %0 = cir.const #cir.fp<0x7F800000> : !cir.float
- %1 = cir.unary(plus, %0) : !cir.float, !cir.float
- cir.return %1 : !cir.float
- }
- // CHECK: cir.func{{.*}} @unary_plus_nan() -> !cir.float
- // CHECK-NEXT: %[[CONST:.*]] = cir.const #cir.fp<0x7F800000> : !cir.float
- // CHECK-NEXT: cir.return %[[CONST]] : !cir.float
-
- cir.func @unary_plus_neg_nan() -> !cir.float {
- %0 = cir.const #cir.fp<0xFF800000> : !cir.float
- %1 = cir.unary(plus, %0) : !cir.float, !cir.float
- cir.return %1 : !cir.float
- }
- // CHECK: cir.func{{.*}} @unary_plus_neg_nan() -> !cir.float
- // CHECK-NEXT: %[[CONST:.*]] = cir.const #cir.fp<0xFF800000> : !cir.float
- // CHECK-NEXT: cir.return %[[CONST]] : !cir.float
-
cir.func @unary_minus_true() -> !cir.bool {
%0 = cir.const #true
%1 = cir.unary(minus, %0) : !cir.bool, !cir.bool
More information about the llvm-branch-commits
mailing list