[llvm] [SLP][REVEC] Enable vectorisation of vector_extract intrinsics (PR #148820)

[Gaëtan Bossu via llvm-commits]

https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/148820

>From 4e3b769bc9c28b60e429186ef956b6cc16d6db51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 24 Jun 2025 12:35:19 +0000
Subject: [PATCH 1/2] [SLP] Pre-commit revec tests for vector_extract
 intrinsics

The SLPVectorizer knows how to look through extractelement instructions,
but not vector_extract intrinsics. Support will be added in future
commit.
---
 .../SLPVectorizer/revec-extractvector.ll      | 355 ++++++++++++++++++
 1 file changed, 355 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/revec-extractvector.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/revec-extractvector.ll b/llvm/test/Transforms/SLPVectorizer/revec-extractvector.ll
new file mode 100644
index 0000000000000..5942b0636c909
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/revec-extractvector.ll
@@ -0,0 +1,355 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -S -slp-revec -slp-max-reg-size=256 %s | FileCheck %s
+
+define i32 @test_32xi8_2parts(ptr %in, ptr %out) {
+; CHECK-LABEL: @test_32xi8_2parts(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[IN:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8> [[TMP0]], i64 16)
+; CHECK-NEXT:    [[ADD_1:%.*]] = add <16 x i8> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = add <16 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[OUT_2:%.*]] = getelementptr inbounds i8, ptr [[OUT:%.*]], i64 16
+; CHECK-NEXT:    store <16 x i8> [[ADD_1]], ptr [[OUT]], align 16
+; CHECK-NEXT:    store <16 x i8> [[ADD_2]], ptr [[OUT_2]], align 16
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load <32 x i8>, ptr %in, align 1
+  %1 = call <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8> %0, i64 0)
+  %2 = call <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8> %0, i64 16)
+
+  %add.1 = add <16 x i8> %1, %1
+  %add.2 = add <16 x i8> %2, %2
+
+  %out.2 = getelementptr inbounds i8, ptr %out, i64 16
+  store <16 x i8> %add.1, ptr %out, align 16
+  store <16 x i8> %add.2, ptr %out.2, align 16
+  ret i32 0
+}
+
+define i32 @test_32xi8_4_parts(ptr %in, ptr %out) {
+; CHECK-LABEL: @test_32xi8_4_parts(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[IN:%.*]], align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 8)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 16)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 24)
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP9]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> [[TMP6]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i8> [[TMP7]], <32 x i8> [[TMP11]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT:    [[TMP1:%.*]] = add <32 x i8> [[TMP8]], [[TMP8]]
+; CHECK-NEXT:    store <32 x i8> [[TMP1]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load <32 x i8>, ptr %in, align 1
+  %1 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 0)
+  %2 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 8)
+  %3 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 16)
+  %4 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 24)
+
+  %add.1 = add <8 x i8> %1, %1
+  %add.2 = add <8 x i8> %2, %2
+  %add.3 = add <8 x i8> %3, %3
+  %add.4 = add <8 x i8> %4, %4
+
+  store <8 x i8> %add.1, ptr %out, align 8
+  %out.2 = getelementptr inbounds i8, ptr %out, i64 8
+  store <8 x i8> %add.2, ptr %out.2, align 8
+  %out.3 = getelementptr inbounds i8, ptr %out, i64 16
+  store <8 x i8> %add.3, ptr %out.3, align 8
+  %out.4 = getelementptr inbounds i8, ptr %out, i64 24
+  store <8 x i8> %add.4, ptr %out.4, align 8
+  ret i32 0
+}
+
+; Extract the 1st, 2nd and 4th quarters of a <32 x i8> register
+; The 3rd quarter comes from a %vec argument.
+; We are expecting a wide load followed by a vector.insert.
+define i32 @test_32xi8_4_parts_sparse(ptr %in, ptr %out, <8 x i8> %vec) {
+; CHECK-LABEL: @test_32xi8_4_parts_sparse(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[IN:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 8)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 24)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[VEC:%.*]], <8 x i8> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP7]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> [[TMP5]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i8> [[TMP8]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[TMP11]], <32 x i8> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <32 x i8> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    store <32 x i8> [[TMP2]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load <32 x i8>, ptr %in, align 1
+  %1 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 0)
+  %2 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 8)
+  %4 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 24)
+
+  %add.1 = add <8 x i8> %1, %1
+  %add.2 = add <8 x i8> %2, %2
+  %add.3 = add <8 x i8> %vec, %vec
+  %add.4 = add <8 x i8> %4, %4
+
+  store <8 x i8> %add.1, ptr %out, align 8
+  %out.2 = getelementptr inbounds i8, ptr %out, i64 8
+  store <8 x i8> %add.2, ptr %out.2, align 8
+  %out.3 = getelementptr inbounds i8, ptr %out, i64 16
+  store <8 x i8> %add.3, ptr %out.3, align 8
+  %out.4 = getelementptr inbounds i8, ptr %out, i64 24
+  store <8 x i8> %add.4, ptr %out.4, align 8
+  ret i32 0
+}
+
+; Extract the 1st and 3rd quarters of a <32 x i8> register
+define i32 @test_32xi8_4_parts_sparse2(ptr %in, ptr %out) {
+; CHECK-LABEL: @test_32xi8_4_parts_sparse2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[IN:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 16)
+; CHECK-NEXT:    [[ADD_1:%.*]] = add <8 x i8> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = add <8 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    store <8 x i8> [[ADD_1]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[OUT_2:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 8
+; CHECK-NEXT:    store <8 x i8> [[ADD_2]], ptr [[OUT_2]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load <32 x i8>, ptr %in, align 1
+  %1 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 0)
+  %2 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> %0, i64 16)
+
+  %add.1 = add <8 x i8> %1, %1
+  %add.2 = add <8 x i8> %2, %2
+
+  store <8 x i8> %add.1, ptr %out, align 8
+  %out.2 = getelementptr inbounds i8, ptr %out, i64 8
+  store <8 x i8> %add.2, ptr %out.2, align 8
+  ret i32 0
+}
+
+; Vector size is 256-bit, we should generate 2 <16 x i16> stores.
+define i32 @test_32xi16_4_parts(ptr %in, ptr %out) {
+; CHECK-LABEL: @test_32xi16_4_parts(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i16>, ptr [[IN:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> [[TMP0]], i64 8)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> [[TMP0]], i64 16)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> [[TMP0]], i64 24)
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <32 x i16> [[TMP5]], <32 x i16> [[TMP6]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i16> [[TMP7]], <32 x i16> [[TMP11]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i16> [[TMP12]], <32 x i16> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT:    [[TMP9:%.*]] = add <32 x i16> [[TMP8]], [[TMP8]]
+; CHECK-NEXT:    store <32 x i16> [[TMP9]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load <32 x i16>, ptr %in, align 1
+  %1 = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> %0, i64 0)
+  %2 = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> %0, i64 8)
+  %3 = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> %0, i64 16)
+  %4 = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> %0, i64 24)
+
+  %add.1 = add <8 x i16> %1, %1
+  %add.2 = add <8 x i16> %2, %2
+  %add.3 = add <8 x i16> %3, %3
+  %add.4 = add <8 x i16> %4, %4
+
+  store <8 x i16> %add.1, ptr %out, align 8
+  %out.2 = getelementptr inbounds i16, ptr %out, i64 8
+  store <8 x i16> %add.2, ptr %out.2, align 8
+  %out.3 = getelementptr inbounds i16, ptr %out, i64 16
+  store <8 x i16> %add.3, ptr %out.3, align 8
+  %out.4 = getelementptr inbounds i16, ptr %out, i64 24
+  store <8 x i16> %add.4, ptr %out.4, align 8
+  ret i32 0
+}
+
+; Extract lo/hi halves from two different source vectors.
+define i32 @test_2x_16xi8(ptr %in0, ptr %in1, ptr %out) {
+; CHECK-LABEL: @test_2x_16xi8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = load <16 x i8>, ptr [[IN0:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE0]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE0]], i64 8)
+; CHECK-NEXT:    [[BASE1:%.*]] = load <16 x i8>, ptr [[IN1:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE1]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE1]], i64 8)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[TMP7]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP8]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> [[TMP5]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <32 x i8> [[TMP11]], <32 x i8> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT:    [[TMP1:%.*]] = add <32 x i8> [[TMP0]], [[TMP0]]
+; CHECK-NEXT:    store <32 x i8> [[TMP1]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %base0 = load <16 x i8>, ptr %in0, align 1
+  %1 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base0, i64 0)
+  %2 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base0, i64 8)
+
+  %base1 = load <16 x i8>, ptr %in1, align 1
+  %3 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base1, i64 0)
+  %4 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base1, i64 8)
+
+  %add.1 = add <8 x i8> %1, %1
+  %add.2 = add <8 x i8> %2, %2
+  %add.3 = add <8 x i8> %3, %3
+  %add.4 = add <8 x i8> %4, %4
+
+  store <8 x i8> %add.1, ptr %out, align 8
+  %out.2 = getelementptr inbounds i8, ptr %out, i64 8
+  store <8 x i8> %add.2, ptr %out.2, align 8
+  %out.3 = getelementptr inbounds i8, ptr %out, i64 16
+  store <8 x i8> %add.3, ptr %out.3, align 8
+  %out.4 = getelementptr inbounds i8, ptr %out, i64 24
+  store <8 x i8> %add.4, ptr %out.4, align 8
+  ret i32 0
+}
+
+; Extract lo/hi halves from one vector, and hi from another one.
+; The hilo quarter comes from a function argument.
+; We are expecting a shuffle of two sources, followed by a vector.insert.
+define i32 @test_2x_16xi8_sparse(ptr %in0, ptr %in1, ptr %out, <8 x i8> %vec) {
+; CHECK-LABEL: @test_2x_16xi8_sparse(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE1:%.*]] = load <16 x i8>, ptr [[IN1:%.*]], align 1
+; CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE1]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE1]], i64 8)
+; CHECK-NEXT:    [[BASE2:%.*]] = load <16 x i8>, ptr [[IN2:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE2]], i64 8)
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i8> [[VEC:%.*]], <8 x i8> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[TMP0]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP3]], <32 x i8> [[TMP4]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP6]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i8> [[TMP7]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[TMP10]], <32 x i8> [[TMP8]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <32 x i8> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    store <32 x i8> [[TMP2]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %base0 = load <16 x i8>, ptr %in0, align 1
+  %1 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base0, i64 0)
+  %2 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base0, i64 8)
+
+  %base1 = load <16 x i8>, ptr %in1, align 1
+  ; %3 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base1, i64 0)
+  %4 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base1, i64 8)
+
+  %add.1 = add <8 x i8> %1, %1
+  %add.2 = add <8 x i8> %2, %2
+  %add.3 = add <8 x i8> %vec, %vec
+  %add.4 = add <8 x i8> %4, %4
+
+  store <8 x i8> %add.1, ptr %out, align 8
+  %out.2 = getelementptr inbounds i8, ptr %out, i64 8
+  store <8 x i8> %add.2, ptr %out.2, align 8
+  %out.3 = getelementptr inbounds i8, ptr %out, i64 16
+  store <8 x i8> %add.3, ptr %out.3, align 8
+  %out.4 = getelementptr inbounds i8, ptr %out, i64 24
+  store <8 x i8> %add.4, ptr %out.4, align 8
+  ret i32 0
+}
+
+; Extract lo/hi halves from four different source vectors.
+; This is not supported as we are looking for a Gather TreeEntry of
+; vector.extract instructions with a maximum of two unique inputs.
+; There are four here.
+define i32 @test_4x_8xi8(ptr %in0, ptr %in1, ptr %in2, ptr %in3, ptr %out) {
+; CHECK-LABEL: @test_4x_8xi8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = load <8 x i8>, ptr [[IN0:%.*]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE0]], i64 0)
+; CHECK-NEXT:    [[TMP12:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE0]], i64 4)
+; CHECK-NEXT:    [[BASE2:%.*]] = load <8 x i8>, ptr [[IN2:%.*]], align 1
+; CHECK-NEXT:    [[TMP13:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE2]], i64 0)
+; CHECK-NEXT:    [[TMP14:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE2]], i64 4)
+; CHECK-NEXT:    [[BASE4:%.*]] = load <8 x i8>, ptr [[IN4:%.*]], align 1
+; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE4]], i64 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE4]], i64 4)
+; CHECK-NEXT:    [[BASE3:%.*]] = load <8 x i8>, ptr [[IN3:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE3]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE3]], i64 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP12]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP22]], <32 x i8> [[TMP23]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 32, i32 33, i32 34, i32 35, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <32 x i8> [[TMP10]], <32 x i8> [[TMP24]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i8> [[TMP14]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <32 x i8> [[TMP25]], <32 x i8> [[TMP26]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 32, i32 33, i32 34, i32 35, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP27]], <32 x i8> [[TMP15]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> [[TMP17]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 32, i32 33, i32 34, i32 35, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <32 x i8> [[TMP18]], <32 x i8> [[TMP19]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP20]], <32 x i8> [[TMP21]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEXT:    [[TMP9:%.*]] = add <32 x i8> [[TMP8]], [[TMP8]]
+; CHECK-NEXT:    store <32 x i8> [[TMP9]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %base0 = load <8 x i8>, ptr %in0, align 1
+  %1 = call <4 x i8> @llvm.vector.extract.v8i8.v32i8(<8 x i8> %base0, i64 0)
+  %2 = call <4 x i8> @llvm.vector.extract.v8i8.v32i8(<8 x i8> %base0, i64 4)
+
+  %base1 = load <8 x i8>, ptr %in1, align 1
+  %3 = call <4 x i8> @llvm.vector.extract.v8i8.v32i8(<8 x i8> %base1, i64 0)
+  %4 = call <4 x i8> @llvm.vector.extract.v8i8.v32i8(<8 x i8> %base1, i64 4)
+
+  %base2 = load <8 x i8>, ptr %in2, align 1
+  %5 = call <4 x i8> @llvm.vector.extract.v8i8.v32i8(<8 x i8> %base2, i64 0)
+  %6 = call <4 x i8> @llvm.vector.extract.v8i8.v32i8(<8 x i8> %base2, i64 4)
+
+  %base3 = load <8 x i8>, ptr %in3, align 1
+  %7 = call <4 x i8> @llvm.vector.extract.v8i8.v32i8(<8 x i8> %base3, i64 0)
+  %8 = call <4 x i8> @llvm.vector.extract.v8i8.v32i8(<8 x i8> %base3, i64 4)
+
+  %add.1 = add <4 x i8> %1, %1
+  %add.2 = add <4 x i8> %2, %2
+  %add.3 = add <4 x i8> %3, %3
+  %add.4 = add <4 x i8> %4, %4
+  %add.5 = add <4 x i8> %5, %5
+  %add.6 = add <4 x i8> %6, %6
+  %add.7 = add <4 x i8> %7, %7
+  %add.8 = add <4 x i8> %8, %8
+
+  store <4 x i8> %add.1, ptr %out, align 8
+  %out.2 = getelementptr inbounds i8, ptr %out, i64 4
+  store <4 x i8> %add.2, ptr %out.2, align 8
+  %out.3 = getelementptr inbounds i8, ptr %out, i64 8
+  store <4 x i8> %add.3, ptr %out.3, align 8
+  %out.4 = getelementptr inbounds i8, ptr %out, i64 12
+  store <4 x i8> %add.4, ptr %out.4, align 8
+  %out.5 = getelementptr inbounds i8, ptr %out, i64 16
+  store <4 x i8> %add.5, ptr %out.5, align 8
+  %out.6 = getelementptr inbounds i8, ptr %out, i64 20
+  store <4 x i8> %add.6, ptr %out.6, align 8
+  %out.7 = getelementptr inbounds i8, ptr %out, i64 24
+  store <4 x i8> %add.7, ptr %out.7, align 8
+  %out.8 = getelementptr inbounds i8, ptr %out, i64 28
+  store <4 x i8> %add.8, ptr %out.8, align 8
+  ret i32 0
+}

>From 85707da39b18fcfc352eda68f9741a8868edf96d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 15 Jul 2025 13:06:37 +0000
Subject: [PATCH 2/2] [SLP] Look through vector_extract intrinsics for gather
 nodes

This is mostly a manual change to match more instructions, but one needs
to be careful with masks, as the "elements" being shuffled are vectors.
This means element indices sometimes need to be converted to "subvector"
indices.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 185 +++++++++++-------
 .../SLPVectorizer/revec-extractvector.ll      | 120 +++---------
 2 files changed, 144 insertions(+), 161 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 31aec77db63c1..d36173211e7a8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -727,6 +727,31 @@ static SmallBitVector isUndefVector(const Value *V,
   return Res;
 }
 
+struct ExtractFromVector {
+  Value *IndexOperand;
+  Value *VectorOperand;
+
+  std::optional<unsigned> getConstantIndex() const {
+    if (auto *CI = dyn_cast<ConstantInt>(IndexOperand))
+      return CI->getZExtValue();
+    return {};
+  }
+
+  VectorType *getVectorOperandType() const {
+    return cast<VectorType>(VectorOperand->getType());
+  }
+};
+
+/// Match ExtractElementInst or Intrinsic::vector_extract
+static std::optional<ExtractFromVector> matchExtractFromVector(Value *V) {
+  if (auto *EI = dyn_cast<ExtractElementInst>(V))
+    return ExtractFromVector{EI->getIndexOperand(), EI->getVectorOperand()};
+  if (auto *IntrI = dyn_cast<IntrinsicInst>(V);
+      IntrI && IntrI->getIntrinsicID() == Intrinsic::vector_extract)
+    return ExtractFromVector{IntrI->getOperand(1), IntrI->getOperand(0)};
+  return {};
+}
+
 /// Checks if the vector of instructions can be represented as a shuffle, like:
 /// %x0 = extractelement <4 x i8> %x, i32 0
 /// %x3 = extractelement <4 x i8> %x, i32 3
@@ -752,42 +777,47 @@ static SmallBitVector isUndefVector(const Value *V,
 static std::optional<TargetTransformInfo::ShuffleKind>
 isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
                      AssumptionCache *AC) {
-  const auto *It = find_if(VL, IsaPred<ExtractElementInst>);
-  if (It == VL.end())
-    return std::nullopt;
-  unsigned Size =
+  unsigned ShuffleSrcSize =
       std::accumulate(VL.begin(), VL.end(), 0u, [](unsigned S, Value *V) {
-        auto *EI = dyn_cast<ExtractElementInst>(V);
-        if (!EI)
+        std::optional<ExtractFromVector> EFV = matchExtractFromVector(V);
+        if (!EFV)
           return S;
-        auto *VTy = dyn_cast<FixedVectorType>(EI->getVectorOperandType());
+        auto *VTy = dyn_cast<FixedVectorType>(EFV->getVectorOperandType());
         if (!VTy)
           return S;
         return std::max(S, VTy->getNumElements());
       });
+  if (ShuffleSrcSize == 0)
+    return std::nullopt;
 
   Value *Vec1 = nullptr;
   Value *Vec2 = nullptr;
   bool HasNonUndefVec = any_of(VL, [&](Value *V) {
-    auto *EE = dyn_cast<ExtractElementInst>(V);
-    if (!EE)
+    std::optional<ExtractFromVector> EFV = matchExtractFromVector(V);
+    if (!EFV)
       return false;
-    Value *Vec = EE->getVectorOperand();
+    Value *Vec = EFV->VectorOperand;
     if (isa<UndefValue>(Vec))
       return false;
     return isGuaranteedNotToBePoison(Vec, AC);
   });
   enum ShuffleMode { Unknown, Select, Permute };
   ShuffleMode CommonShuffleMode = Unknown;
+  Type *ExtractedTy = VL[0]->getType();
+  unsigned EltsPerExtractInst = getNumElements(ExtractedTy);
+
+  // Note: Mask is for values of VL, which can be of vector type.
   Mask.assign(VL.size(), PoisonMaskElem);
+
   for (unsigned I = 0, E = VL.size(); I < E; ++I) {
     // Undef can be represented as an undef element in a vector.
     if (isa<UndefValue>(VL[I]))
       continue;
-    auto *EI = cast<ExtractElementInst>(VL[I]);
-    if (isa<ScalableVectorType>(EI->getVectorOperandType()))
+    std::optional<ExtractFromVector> EFV = matchExtractFromVector(VL[I]);
+    assert(EFV.has_value() && "Unexpected shuffle source.");
+    if (isa<ScalableVectorType>(EFV->getVectorOperandType()))
       return std::nullopt;
-    auto *Vec = EI->getVectorOperand();
+    auto *Vec = EFV->VectorOperand;
     // We can extractelement from undef or poison vector.
     if (isUndefVector</*isPoisonOnly=*/true>(Vec).all())
       continue;
@@ -795,16 +825,15 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
     if (isa<UndefValue>(Vec)) {
       Mask[I] = I;
     } else {
-      if (isa<UndefValue>(EI->getIndexOperand()))
+      if (isa<UndefValue>(EFV->IndexOperand))
         continue;
-      auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
-      if (!Idx)
+      std::optional<unsigned> Idx = EFV->getConstantIndex();
+      if (!Idx || *Idx % EltsPerExtractInst != 0)
         return std::nullopt;
       // Undefined behavior if Idx is negative or >= Size.
-      if (Idx->getValue().uge(Size))
+      if (*Idx >= ShuffleSrcSize)
         continue;
-      unsigned IntIdx = Idx->getValue().getZExtValue();
-      Mask[I] = IntIdx;
+      Mask[I] = *Idx / EltsPerExtractInst;
     }
     if (isUndefVector(Vec).all() && HasNonUndefVec)
       continue;
@@ -814,7 +843,7 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
       Vec1 = Vec;
     } else if (!Vec2 || Vec2 == Vec) {
       Vec2 = Vec;
-      Mask[I] += Size;
+      Mask[I] += (ShuffleSrcSize / EltsPerExtractInst);
     } else {
       return std::nullopt;
     }
@@ -822,7 +851,7 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
       continue;
     // If the extract index is not the same as the operation number, it is a
     // permutation.
-    if (Mask[I] % Size != I) {
+    if ((Mask[I] * EltsPerExtractInst) % ShuffleSrcSize != I) {
       CommonShuffleMode = Permute;
       continue;
     }
@@ -12068,22 +12097,30 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
   /// Compute the cost of creating a vector containing the extracted values from
   /// \p VL.
   InstructionCost
-  computeExtractCost(ArrayRef<Value *> VL, ArrayRef<int> Mask,
+  computeExtractCost(ArrayRef<Value *> VL, ArrayRef<int> MaskRef,
                      ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
                      unsigned NumParts) {
     assert(VL.size() > NumParts && "Unexpected scalarized shuffle.");
     unsigned NumElts =
         std::accumulate(VL.begin(), VL.end(), 0, [](unsigned Sz, Value *V) {
-          auto *EE = dyn_cast<ExtractElementInst>(V);
-          if (!EE)
+          std::optional<ExtractFromVector> EFV = matchExtractFromVector(V);
+          if (!EFV)
             return Sz;
-          auto *VecTy = dyn_cast<FixedVectorType>(EE->getVectorOperandType());
+          auto *VecTy = dyn_cast<FixedVectorType>(EFV->getVectorOperandType());
           if (!VecTy)
             return Sz;
           return std::max(Sz, VecTy->getNumElements());
         });
     // FIXME: this must be moved to TTI for better estimation.
-    unsigned EltsPerVector = getPartNumElems(VL.size(), NumParts);
+    unsigned EltsPerExtractInst = getNumElements(VL[0]->getType());
+    unsigned EltsPerVector =
+        getPartNumElems(VL.size() * EltsPerExtractInst, NumParts);
+
+    // Make sure we get a proper shuffle mask if the elements being extracted
+    // are subvectors.
+    SmallVector<int> Mask(MaskRef.begin(), MaskRef.end());
+    transformScalarShuffleIndiciesToVector(EltsPerExtractInst, Mask);
+
     auto CheckPerRegistersShuffle = [&](MutableArrayRef<int> Mask,
                                         SmallVectorImpl<unsigned> &Indices,
                                         SmallVectorImpl<unsigned> &SubVecSizes)
@@ -12150,7 +12187,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     for (unsigned Part : seq<unsigned>(NumParts)) {
       if (!ShuffleKinds[Part])
         continue;
-      ArrayRef<int> MaskSlice = Mask.slice(
+      ArrayRef<int> MaskSlice = ArrayRef<int>(Mask).slice(
           Part * EltsPerVector, getNumElems(Mask.size(), EltsPerVector, Part));
       SmallVector<int> SubMask(EltsPerVector, PoisonMaskElem);
       copy(MaskSlice, SubMask.begin());
@@ -12555,6 +12592,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
         });
     SmallPtrSet<Value *, 4> UniqueBases;
     unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
+    Type *ExtractedTy = VL[0]->getType();
+    const unsigned EltsPerExtractInst = getNumElements(ExtractedTy);
     SmallDenseMap<Value *, APInt, 4> VectorOpsToExtracts;
     for (unsigned Part : seq<unsigned>(NumParts)) {
       unsigned Limit = getNumElems(VL.size(), SliceSize, Part);
@@ -12571,13 +12610,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
         // vectorized tree.
         // Also, avoid adjusting the cost for extractelements with multiple uses
         // in different graph entries.
-        auto *EE = cast<ExtractElementInst>(V);
-        VecBase = EE->getVectorOperand();
+        ExtractFromVector EFV = matchExtractFromVector(V).value();
+        Instruction *ExtractInst = cast<Instruction>(V);
+        VecBase = EFV.VectorOperand;
         UniqueBases.insert(VecBase);
         ArrayRef<TreeEntry *> VEs = R.getTreeEntries(V);
         if (!CheckedExtracts.insert(V).second ||
             !R.areAllUsersVectorized(cast<Instruction>(V), &VectorizedVals) ||
-            any_of(EE->users(),
+            any_of(V->users(),
                    [&](User *U) {
                      return isa<GetElementPtrInst>(U) &&
                             !R.areAllUsersVectorized(cast<Instruction>(U),
@@ -12585,39 +12625,45 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
                    }) ||
             (!VEs.empty() && !is_contained(VEs, E)))
           continue;
-        std::optional<unsigned> EEIdx = getExtractIndex(EE);
+        std::optional<unsigned> EEIdx = EFV.getConstantIndex();
         if (!EEIdx)
           continue;
         unsigned Idx = *EEIdx;
         // Take credit for instruction that will become dead.
-        if (EE->hasOneUse() || !PrevNodeFound) {
-          Instruction *Ext = EE->user_back();
-          if (isa<SExtInst, ZExtInst>(Ext) &&
+        if (V->hasOneUse() || !PrevNodeFound) {
+          Instruction *Ext = ExtractInst->user_back();
+          if (isa<SExtInst, ZExtInst>(Ext) && !V->getType()->isVectorTy() &&
               all_of(Ext->users(), IsaPred<GetElementPtrInst>)) {
             // Use getExtractWithExtendCost() to calculate the cost of
             // extractelement/ext pair.
             Cost -= TTI.getExtractWithExtendCost(
-                Ext->getOpcode(), Ext->getType(), EE->getVectorOperandType(),
+                Ext->getOpcode(), Ext->getType(), EFV.getVectorOperandType(),
                 Idx, CostKind);
             // Add back the cost of s|zext which is subtracted separately.
             Cost += TTI.getCastInstrCost(
-                Ext->getOpcode(), Ext->getType(), EE->getType(),
+                Ext->getOpcode(), Ext->getType(), V->getType(),
                 TTI::getCastContextHint(Ext), CostKind, Ext);
             continue;
           }
         }
+
+        // Note: If extracting a subvector, NumExtractedElts will be >1, and the
+        // bit set in DemandedElts will correspond to a sub-vector index.
         APInt &DemandedElts =
             VectorOpsToExtracts
                 .try_emplace(VecBase,
-                             APInt::getZero(getNumElements(VecBase->getType())))
+                             APInt::getZero(getNumElements(VecBase->getType()) /
+                                            EltsPerExtractInst))
                 .first->getSecond();
-        DemandedElts.setBit(Idx);
+        DemandedElts.setBit(Idx / EltsPerExtractInst);
       }
     }
+
     for (const auto &[Vec, DemandedElts] : VectorOpsToExtracts)
-      Cost -= TTI.getScalarizationOverhead(cast<VectorType>(Vec->getType()),
-                                           DemandedElts, /*Insert=*/false,
-                                           /*Extract=*/true, CostKind);
+      Cost -= getScalarizationOverhead(TTI, VL[0]->getType(),
+                                       cast<VectorType>(Vec->getType()),
+                                       DemandedElts, /*Insert=*/false,
+                                       /*Extract=*/true, CostKind);
     // Check that gather of extractelements can be represented as just a
     // shuffle of a single/two vectors the scalars are extracted from.
     // Found the bunch of extractelement instructions that must be gathered
@@ -12704,11 +12750,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
                   [&](auto P) {
                     if (P.value() == PoisonMaskElem)
                       return Mask[P.index()] == PoisonMaskElem;
-                    auto *EI = cast<ExtractElementInst>(
-                        cast<const TreeEntry *>(InVectors.front())
-                            ->getOrdered(P.index()));
-                    return EI->getVectorOperand() == V1 ||
-                           EI->getVectorOperand() == V2;
+                    ExtractFromVector EFV =
+                        matchExtractFromVector(
+                            cast<const TreeEntry *>(InVectors.front())
+                                ->getOrdered(P.index()))
+                            .value();
+                    return EFV.VectorOperand == V1 || EFV.VectorOperand == V2;
                   }) &&
            "Expected extractelement vectors.");
   }
@@ -12734,8 +12781,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
                                isa<UndefValue>(Scalar);
                       if (isa<Constant>(V1))
                         return true;
-                      auto *EI = cast<ExtractElementInst>(Scalar);
-                      return EI->getVectorOperand() == V1;
+                      ExtractFromVector EFV =
+                          matchExtractFromVector(Scalar).value();
+                      return EFV.VectorOperand == V1;
                     }) &&
              "Expected only tree entry for extractelement vectors.");
       return;
@@ -15117,19 +15165,20 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
     MutableArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) const {
   // Scan list of gathered scalars for extractelements that can be represented
   // as shuffles.
+  // This maps the vectors we gather from to all the lanes they populate in VL.
   MapVector<Value *, SmallVector<int>> VectorOpToIdx;
   SmallVector<int> UndefVectorExtracts;
   for (int I = 0, E = VL.size(); I < E; ++I) {
-    auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
-    if (!EI) {
+    std::optional<ExtractFromVector> Match = matchExtractFromVector(VL[I]);
+    if (!Match) {
       if (isa<UndefValue>(VL[I]))
         UndefVectorExtracts.push_back(I);
       continue;
     }
-    auto *VecTy = dyn_cast<FixedVectorType>(EI->getVectorOperandType());
-    if (!VecTy || !isa<ConstantInt, UndefValue>(EI->getIndexOperand()))
+    auto *VecTy = dyn_cast<FixedVectorType>(Match->getVectorOperandType());
+    if (!VecTy || !isa<ConstantInt, UndefValue>(Match->IndexOperand))
       continue;
-    std::optional<unsigned> Idx = getExtractIndex(EI);
+    std::optional<unsigned> Idx = Match->getConstantIndex();
     // Undefined index.
     if (!Idx) {
       UndefVectorExtracts.push_back(I);
@@ -15141,12 +15190,13 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
     }
     SmallBitVector ExtractMask(VecTy->getNumElements(), true);
     ExtractMask.reset(*Idx);
-    if (isUndefVector(EI->getVectorOperand(), ExtractMask).all()) {
+    if (isUndefVector(Match->VectorOperand, ExtractMask).all()) {
       UndefVectorExtracts.push_back(I);
       continue;
     }
-    VectorOpToIdx[EI->getVectorOperand()].push_back(I);
+    VectorOpToIdx[Match->VectorOperand].push_back(I);
   }
+
   // Sort the vector operands by the maximum number of uses in extractelements.
   SmallVector<std::pair<Value *, SmallVector<int>>> Vectors =
       VectorOpToIdx.takeVector();
@@ -15160,12 +15210,12 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
   if (!Vectors.empty()) {
     SingleMax = Vectors.front().second.size() + UndefSz;
     if (Vectors.size() > 1) {
-      auto *ItNext = std::next(Vectors.begin());
-      PairMax = SingleMax + ItNext->second.size();
+      PairMax = SingleMax + Vectors[1].second.size();
     }
   }
   if (SingleMax == 0 && PairMax == 0 && UndefSz == 0)
     return std::nullopt;
+
   // Check if better to perform a shuffle of 2 vectors or just of a single
   // vector.
   SmallVector<Value *> SavedVL(VL.begin(), VL.end());
@@ -15182,6 +15232,7 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
   // Add extracts from undefs too.
   for (int Idx : UndefVectorExtracts)
     std::swap(GatheredExtracts[Idx], VL[Idx]);
+
   // Check that gather of extractelements can be represented as just a
   // shuffle of a single/two vectors the scalars are extracted from.
   std::optional<TTI::ShuffleKind> Res =
@@ -15198,13 +15249,7 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
     if (Mask[I] == PoisonMaskElem && !isa<PoisonValue>(GatheredExtracts[I]) &&
         isa<UndefValue>(GatheredExtracts[I])) {
       std::swap(VL[I], GatheredExtracts[I]);
-      continue;
     }
-    auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
-    if (!EI || !isa<FixedVectorType>(EI->getVectorOperandType()) ||
-        !isa<ConstantInt, UndefValue>(EI->getIndexOperand()) ||
-        is_contained(UndefVectorExtracts, I))
-      continue;
   }
   return Res;
 }
@@ -16556,8 +16601,9 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
       int Idx = Mask[I];
       if (Idx == PoisonMaskElem)
         continue;
-      auto *EI = cast<ExtractElementInst>(VL[I]);
-      VecBase = EI->getVectorOperand();
+      ExtractFromVector EFV = matchExtractFromVector(VL[I]).value();
+      Instruction *EI = cast<Instruction>(VL[I]);
+      VecBase = EFV.VectorOperand;
       if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecBase); !TEs.empty())
         VecBase = TEs.front()->VectorizedValue;
       assert(VecBase && "Expected vectorized value.");
@@ -16611,7 +16657,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
             if (std::get<1>(D) == PoisonMaskElem)
               return S;
             Value *VecOp =
-                cast<ExtractElementInst>(std::get<0>(D))->getVectorOperand();
+                matchExtractFromVector(std::get<0>(D))->VectorOperand;
             if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecOp);
                 !TEs.empty())
               VecOp = TEs.front()->VectorizedValue;
@@ -16623,7 +16669,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
       for (const auto [V, I] : VLMask) {
         if (I == PoisonMaskElem)
           continue;
-        Value *VecOp = cast<ExtractElementInst>(V)->getVectorOperand();
+        Value *VecOp = matchExtractFromVector(V)->VectorOperand;
         if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecOp); !TEs.empty())
           VecOp = TEs.front()->VectorizedValue;
         assert(VecOp && "Expected vectorized value.");
@@ -17073,7 +17119,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
         if (I == PoisonMaskElem)
           continue;
         if (ArrayRef<TreeEntry *> TEs = getTreeEntries(
-                cast<ExtractElementInst>(StoredGS[Idx])->getVectorOperand());
+                matchExtractFromVector(StoredGS[Idx])->VectorOperand);
             !TEs.empty())
           ExtractEntries.append(TEs.begin(), TEs.end());
       }
@@ -17280,8 +17326,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
             continue;
           if (isa<UndefValue>(StoredGS[I]))
             continue;
-          auto *EI = cast<ExtractElementInst>(StoredGS[I]);
-          Value *VecOp = EI->getVectorOperand();
+          Value *VecOp = matchExtractFromVector(StoredGS[I])->VectorOperand;
           if (ArrayRef<TreeEntry *> TEs = getTreeEntries(VecOp);
               !TEs.empty() && TEs.front()->VectorizedValue)
             VecOp = TEs.front()->VectorizedValue;
diff --git a/llvm/test/Transforms/SLPVectorizer/revec-extractvector.ll b/llvm/test/Transforms/SLPVectorizer/revec-extractvector.ll
index 5942b0636c909..423ea26dd1545 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec-extractvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec-extractvector.ll
@@ -5,13 +5,8 @@ define i32 @test_32xi8_2parts(ptr %in, ptr %out) {
 ; CHECK-LABEL: @test_32xi8_2parts(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[IN:%.*]], align 1
-; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8> [[TMP0]], i64 0)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8> [[TMP0]], i64 16)
-; CHECK-NEXT:    [[ADD_1:%.*]] = add <16 x i8> [[TMP1]], [[TMP1]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add <16 x i8> [[TMP2]], [[TMP2]]
-; CHECK-NEXT:    [[OUT_2:%.*]] = getelementptr inbounds i8, ptr [[OUT:%.*]], i64 16
-; CHECK-NEXT:    store <16 x i8> [[ADD_1]], ptr [[OUT]], align 16
-; CHECK-NEXT:    store <16 x i8> [[ADD_2]], ptr [[OUT_2]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = add <32 x i8> [[TMP0]], [[TMP0]]
+; CHECK-NEXT:    store <32 x i8> [[TMP1]], ptr [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -32,18 +27,7 @@ define i32 @test_32xi8_4_parts(ptr %in, ptr %out) {
 ; CHECK-LABEL: @test_32xi8_4_parts(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[IN:%.*]], align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 0)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 8)
-; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 16)
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 24)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP9]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> [[TMP6]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i8> [[TMP7]], <32 x i8> [[TMP11]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT:    [[TMP1:%.*]] = add <32 x i8> [[TMP8]], [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add <32 x i8> [[TMP0]], [[TMP0]]
 ; CHECK-NEXT:    store <32 x i8> [[TMP1]], ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -71,21 +55,13 @@ entry:
 
 ; Extract the 1st, 2nd and 4th quarters of a <32 x i8> register
 ; The 3rd quarter comes from a %vec argument.
-; We are expecting a wide load followed by a vector.insert.
+; We are expecting a wide load followed by a vector.insert
 define i32 @test_32xi8_4_parts_sparse(ptr %in, ptr %out, <8 x i8> %vec) {
 ; CHECK-LABEL: @test_32xi8_4_parts_sparse(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[IN:%.*]], align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 0)
-; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 8)
-; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 24)
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[VEC:%.*]], <8 x i8> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP7]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> [[TMP5]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i8> [[TMP8]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[TMP11]], <32 x i8> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i8> [[VEC:%.*]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[TMP0]], <32 x i8> [[TMP3]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <32 x i8> [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    store <32 x i8> [[TMP2]], ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret i32 0
@@ -116,13 +92,9 @@ define i32 @test_32xi8_4_parts_sparse2(ptr %in, ptr %out) {
 ; CHECK-LABEL: @test_32xi8_4_parts_sparse2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[IN:%.*]], align 1
-; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 0)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<32 x i8> [[TMP0]], i64 16)
-; CHECK-NEXT:    [[ADD_1:%.*]] = add <8 x i8> [[TMP1]], [[TMP1]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add <8 x i8> [[TMP2]], [[TMP2]]
-; CHECK-NEXT:    store <8 x i8> [[ADD_1]], ptr [[OUT:%.*]], align 8
-; CHECK-NEXT:    [[OUT_2:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 8
-; CHECK-NEXT:    store <8 x i8> [[ADD_2]], ptr [[OUT_2]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[TMP0]], <32 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <16 x i8> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -144,19 +116,13 @@ define i32 @test_32xi16_4_parts(ptr %in, ptr %out) {
 ; CHECK-LABEL: @test_32xi16_4_parts(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i16>, ptr [[IN:%.*]], align 1
-; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> [[TMP0]], i64 0)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> [[TMP0]], i64 8)
-; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> [[TMP0]], i64 16)
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.v32i16(<32 x i16> [[TMP0]], i64 24)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <32 x i16> [[TMP5]], <32 x i16> [[TMP6]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i16> [[TMP7]], <32 x i16> [[TMP11]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i16> [[TMP12]], <32 x i16> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT:    [[TMP9:%.*]] = add <32 x i16> [[TMP8]], [[TMP8]]
-; CHECK-NEXT:    store <32 x i16> [[TMP9]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[TMP0]], <32 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <16 x i16> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    store <16 x i16> [[TMP2]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[OUT_3:%.*]] = getelementptr inbounds i16, ptr [[OUT]], i64 16
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP0]], <32 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <16 x i16> [[TMP3]], [[TMP3]]
+; CHECK-NEXT:    store <16 x i16> [[TMP4]], ptr [[OUT_3]], align 8
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -186,18 +152,8 @@ define i32 @test_2x_16xi8(ptr %in0, ptr %in1, ptr %out) {
 ; CHECK-LABEL: @test_2x_16xi8(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE0:%.*]] = load <16 x i8>, ptr [[IN0:%.*]], align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE0]], i64 0)
-; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE0]], i64 8)
 ; CHECK-NEXT:    [[BASE1:%.*]] = load <16 x i8>, ptr [[IN1:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE1]], i64 0)
-; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE1]], i64 8)
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[TMP7]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP8]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> [[TMP5]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <32 x i8> [[TMP11]], <32 x i8> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[BASE0]], <16 x i8> [[BASE1]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <32 x i8> [[TMP0]], [[TMP0]]
 ; CHECK-NEXT:    store <32 x i8> [[TMP1]], ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret i32 0
@@ -233,17 +189,10 @@ define i32 @test_2x_16xi8_sparse(ptr %in0, ptr %in1, ptr %out, <8 x i8> %vec) {
 ; CHECK-LABEL: @test_2x_16xi8_sparse(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE1:%.*]] = load <16 x i8>, ptr [[IN1:%.*]], align 1
-; CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE1]], i64 0)
-; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE1]], i64 8)
 ; CHECK-NEXT:    [[BASE2:%.*]] = load <16 x i8>, ptr [[IN2:%.*]], align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[BASE2]], i64 8)
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i8> [[VEC:%.*]], <8 x i8> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[TMP0]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP3]], <32 x i8> [[TMP4]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP6]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i8> [[TMP7]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[TMP10]], <32 x i8> [[TMP8]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[BASE1]], <16 x i8> [[BASE2]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i8> [[VEC:%.*]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[TMP0]], <32 x i8> [[TMP3]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <32 x i8> [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    store <32 x i8> [[TMP2]], ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret i32 0
@@ -254,7 +203,6 @@ entry:
   %2 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base0, i64 8)
 
   %base1 = load <16 x i8>, ptr %in1, align 1
-  ; %3 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base1, i64 0)
   %4 = call <8 x i8> @llvm.vector.extract.v8i8.v32i8(<16 x i8> %base1, i64 8)
 
   %add.1 = add <8 x i8> %1, %1
@@ -280,32 +228,22 @@ define i32 @test_4x_8xi8(ptr %in0, ptr %in1, ptr %in2, ptr %in3, ptr %out) {
 ; CHECK-LABEL: @test_4x_8xi8(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE0:%.*]] = load <8 x i8>, ptr [[IN0:%.*]], align 1
-; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE0]], i64 0)
-; CHECK-NEXT:    [[TMP12:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE0]], i64 4)
 ; CHECK-NEXT:    [[BASE2:%.*]] = load <8 x i8>, ptr [[IN2:%.*]], align 1
-; CHECK-NEXT:    [[TMP13:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE2]], i64 0)
-; CHECK-NEXT:    [[TMP14:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE2]], i64 4)
 ; CHECK-NEXT:    [[BASE4:%.*]] = load <8 x i8>, ptr [[IN4:%.*]], align 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE4]], i64 0)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE4]], i64 4)
 ; CHECK-NEXT:    [[BASE3:%.*]] = load <8 x i8>, ptr [[IN3:%.*]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE3]], i64 0)
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> [[BASE3]], i64 4)
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP12]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP22]], <32 x i8> [[TMP23]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 32, i32 33, i32 34, i32 35, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <32 x i8> [[TMP10]], <32 x i8> [[TMP24]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i8> [[TMP14]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <32 x i8> [[TMP25]], <32 x i8> [[TMP26]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 32, i32 33, i32 34, i32 35, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP27]], <32 x i8> [[TMP15]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> [[TMP17]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 32, i32 33, i32 34, i32 35, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <32 x i8> [[TMP18]], <32 x i8> [[TMP19]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP20]], <32 x i8> [[TMP21]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[BASE0]], <8 x i8> [[BASE2]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> [[TMP5]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> [[TMP7]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 32, i32 33, i32 34, i32 35, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> [[TMP13]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP10]], <32 x i8> [[TMP11]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 32, i32 33, i32 34, i32 35>
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <32 x i8> [[TMP8]], [[TMP8]]
 ; CHECK-NEXT:    store <32 x i8> [[TMP9]], ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret i32 0