[llvm] [DAGCombiner] Fold Any_Ext of a truncating build vector (PR #214005)
Usman Nadeem via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 09:56:56 PDT 2026
https://github.com/UsmanNadeem created https://github.com/llvm/llvm-project/pull/214005
By recreating a wider build vector.
Split from https://github.com/llvm/llvm-project/pull/212156 Tests are also updated on top of PR #212156
Some of the AArch64 regressions are due to ReconstructShuffle() not being able to handle the mask for the wider BUILD_VECTOR. Marking this PR as a draft for now.
Change-Id: I0b9c56665369809a32318605c0610c4cdd907ab1
>From a32507b3dec53fff0a5372d284b7c62cead08efb Mon Sep 17 00:00:00 2001
From: Usman Nadeem <mnadeem at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 09:35:20 -0700
Subject: [PATCH] [DAGCombiner] Fold Any_Ext of a truncating build vector
By recreating a wider build vector.
Split from https://github.com/llvm/llvm-project/pull/212156
Tests are also updated on top of PR #212156
Some of the AArch64 regressions are due to ReconstructShuffle() not being able to handle the mask for the wider BUILD_VECTOR. Marking this PR as a draft for now.
Change-Id: I0b9c56665369809a32318605c0610c4cdd907ab1
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 10 +
.../CodeGen/AArch64/combine-storetomstore.ll | 461 ++++++++----------
llvm/test/CodeGen/AArch64/neon-extadd.ll | 152 +++---
llvm/test/CodeGen/AArch64/sext.ll | 64 ++-
.../AArch64/vec3-loads-ext-trunc-stores.ll | 168 ++++---
llvm/test/CodeGen/AArch64/vec_uaddo.ll | 34 +-
llvm/test/CodeGen/AArch64/vec_umulo.ll | 42 +-
llvm/test/CodeGen/AArch64/vector-compress.ll | 7 +-
llvm/test/CodeGen/AArch64/vselect-ext.ll | 18 +-
llvm/test/CodeGen/AArch64/zext.ll | 60 +--
llvm/test/CodeGen/X86/avx512-mask-op.ll | 34 +-
11 files changed, 477 insertions(+), 573 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f5620743d6ed1..1ff17aec9fd9a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -16691,6 +16691,16 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
return SCC;
}
+ // Build vector can be truncating (wider operands). Check if we can do the
+ // build vector in a wider type, instead of truncating and any extending. All
+ // operands have the same type.
+ // ty1 aext(ty2 build_vector()) -> ty1 build_vector() of wider type.
+ if (N0.getOpcode() == ISD::BUILD_VECTOR &&
+ VT.getScalarSizeInBits() <=
+ N0.getOperand(0).getValueType().getScalarSizeInBits() &&
+ (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)))
+ return DAG.getBuildVector(VT, DL, N0->ops());
+
if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
return NewCtPop;
diff --git a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
index 35e1f3ea3020c..bdfd58787437e 100644
--- a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
+++ b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
@@ -160,13 +160,12 @@ define void @test_masked_store_success_v8i16(<8 x i16> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -184,37 +183,31 @@ define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_success_v8i64(<8 x i64> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8i64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: mov b7, v4.b[2]
-; SVE-NEXT: mov b16, v4.b[0]
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: mov x9, #6 // =0x6
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: mov x8, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[3]
-; SVE-NEXT: mov v16.b[4], v4.b[1]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x0, x8, lsl #3]
+; SVE-NEXT: st1d { z3.d }, p1, [x0, x8, lsl #3]
; SVE-NEXT: mov x8, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
; SVE-NEXT: st1d { z1.d }, p3, [x0, x8, lsl #3]
; SVE-NEXT: st1d { z0.d }, p4, [x0]
; SVE-NEXT: ret
@@ -243,13 +236,12 @@ define void @test_masked_store_success_v8f16(<8 x half> %x, ptr %ptr, <8 x i1> %
define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8f32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -267,37 +259,31 @@ define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1>
define void @test_masked_store_success_v8f64(<8 x double> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8f64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: mov b7, v4.b[2]
-; SVE-NEXT: mov b16, v4.b[0]
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: mov x9, #6 // =0x6
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: mov x8, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[3]
-; SVE-NEXT: mov v16.b[4], v4.b[1]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x0, x8, lsl #3]
+; SVE-NEXT: st1d { z3.d }, p1, [x0, x8, lsl #3]
; SVE-NEXT: mov x8, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
; SVE-NEXT: st1d { z1.d }, p3, [x0, x8, lsl #3]
; SVE-NEXT: st1d { z0.d }, p4, [x0]
; SVE-NEXT: ret
@@ -347,35 +333,32 @@ define void @test_masked_store_success_v16i16(<16 x i16> %x, ptr %ptr, <16 x i1>
define void @test_masked_store_success_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v16i32:
; SVE: // %bb.0:
-; SVE-NEXT: mov d5, v4.d[1]
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v6.8b, v4.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll2 v5.8h, v4.16b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
+; SVE-NEXT: mov x8, #12 // =0xc
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: mov x9, #8 // =0x8
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #8 // =0x8
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: zip1 v7.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
-; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: shl v7.4s, v7.4s, #31
; SVE-NEXT: shl v4.4s, v4.4s, #31
; SVE-NEXT: cmpne p1.s, p0/z, z6.s, #0
-; SVE-NEXT: shl v7.4s, v7.4s, #31
-; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: cmpne p2.s, p0/z, z5.s, #0
+; SVE-NEXT: cmpne p3.s, p0/z, z7.s, #0
; SVE-NEXT: cmpne p4.s, p0/z, z4.s, #0
-; SVE-NEXT: cmpne p2.s, p0/z, z7.s, #0
-; SVE-NEXT: cmpne p3.s, p0/z, z5.s, #0
-; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
-; SVE-NEXT: mov x8, #12 // =0xc
-; SVE-NEXT: st1w { z0.s }, p4, [x0]
+; SVE-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: st1w { z2.s }, p2, [x0, x9, lsl #2]
-; SVE-NEXT: st1w { z3.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z1.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: ret
%load = load <16 x i32>, ptr %ptr, align 32
%sel = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %load
@@ -472,64 +455,60 @@ define void @test_masked_store_success_v32i16(<32 x i16> %x, ptr %ptr, <32 x i1>
; SVE-NEXT: fmov s4, w9
; SVE-NEXT: ldr w9, [sp, #8]
; SVE-NEXT: fmov s5, w11
-; SVE-NEXT: mov v7.b[1], w2
+; SVE-NEXT: mov v7.h[1], w2
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
; SVE-NEXT: fmov s6, w9
; SVE-NEXT: ldr w9, [sp, #152]
-; SVE-NEXT: mov v4.b[1], w8
+; SVE-NEXT: mov v4.h[1], w8
; SVE-NEXT: ldr w8, [sp, #16]
-; SVE-NEXT: mov v5.b[1], w10
-; SVE-NEXT: mov v6.b[1], w8
+; SVE-NEXT: mov v5.h[1], w10
+; SVE-NEXT: mov v6.h[1], w8
; SVE-NEXT: ldr w8, [sp, #88]
-; SVE-NEXT: mov v7.b[2], w3
-; SVE-NEXT: mov v4.b[2], w8
+; SVE-NEXT: mov v7.h[2], w3
+; SVE-NEXT: mov v4.h[2], w8
; SVE-NEXT: ldr w8, [sp, #24]
-; SVE-NEXT: mov v5.b[2], w9
+; SVE-NEXT: mov v5.h[2], w9
; SVE-NEXT: ldr w9, [sp, #160]
-; SVE-NEXT: mov v6.b[2], w8
+; SVE-NEXT: mov v6.h[2], w8
; SVE-NEXT: ldr w8, [sp, #96]
-; SVE-NEXT: mov v7.b[3], w4
-; SVE-NEXT: mov v4.b[3], w8
+; SVE-NEXT: mov v7.h[3], w4
+; SVE-NEXT: mov v4.h[3], w8
; SVE-NEXT: ldr w8, [sp, #32]
-; SVE-NEXT: mov v5.b[3], w9
+; SVE-NEXT: mov v5.h[3], w9
; SVE-NEXT: ldr w9, [sp, #168]
-; SVE-NEXT: mov v6.b[3], w8
+; SVE-NEXT: mov v6.h[3], w8
; SVE-NEXT: ldr w8, [sp, #104]
-; SVE-NEXT: mov v7.b[4], w5
-; SVE-NEXT: mov v4.b[4], w8
+; SVE-NEXT: mov v7.h[4], w5
+; SVE-NEXT: mov v4.h[4], w8
; SVE-NEXT: ldr w8, [sp, #40]
-; SVE-NEXT: mov v5.b[4], w9
+; SVE-NEXT: mov v5.h[4], w9
; SVE-NEXT: ldr w9, [sp, #176]
-; SVE-NEXT: mov v6.b[4], w8
+; SVE-NEXT: mov v6.h[4], w8
; SVE-NEXT: ldr w8, [sp, #112]
-; SVE-NEXT: mov v7.b[5], w6
-; SVE-NEXT: mov v4.b[5], w8
+; SVE-NEXT: mov v7.h[5], w6
+; SVE-NEXT: mov v4.h[5], w8
; SVE-NEXT: ldr w8, [sp, #48]
-; SVE-NEXT: mov v5.b[5], w9
+; SVE-NEXT: mov v5.h[5], w9
; SVE-NEXT: ldr w9, [sp, #184]
-; SVE-NEXT: mov v6.b[5], w8
+; SVE-NEXT: mov v6.h[5], w8
; SVE-NEXT: ldr w8, [sp, #120]
-; SVE-NEXT: mov v7.b[6], w7
-; SVE-NEXT: mov v4.b[6], w8
+; SVE-NEXT: mov v7.h[6], w7
+; SVE-NEXT: mov v4.h[6], w8
; SVE-NEXT: ldr w8, [sp, #56]
-; SVE-NEXT: mov v5.b[6], w9
+; SVE-NEXT: mov v5.h[6], w9
; SVE-NEXT: ldr w9, [sp, #192]
-; SVE-NEXT: mov v6.b[6], w8
+; SVE-NEXT: mov v6.h[6], w8
; SVE-NEXT: ldr w8, [sp, #128]
-; SVE-NEXT: mov v4.b[7], w8
+; SVE-NEXT: mov v4.h[7], w8
; SVE-NEXT: ldr w8, [sp, #64]
-; SVE-NEXT: mov v5.b[7], w9
+; SVE-NEXT: mov v5.h[7], w9
; SVE-NEXT: ldr w9, [sp]
-; SVE-NEXT: mov v6.b[7], w8
+; SVE-NEXT: mov v6.h[7], w8
; SVE-NEXT: mov x8, #16 // =0x10
-; SVE-NEXT: mov v7.b[7], w9
-; SVE-NEXT: ushll v4.8h, v4.8b, #0
-; SVE-NEXT: ushll v5.8h, v5.8b, #0
-; SVE-NEXT: ushll v6.8h, v6.8b, #0
-; SVE-NEXT: ushll v7.8h, v7.8b, #0
+; SVE-NEXT: mov v7.h[7], w9
; SVE-NEXT: shl v4.8h, v4.8h, #15
; SVE-NEXT: shl v5.8h, v5.8h, #15
; SVE-NEXT: shl v6.8h, v6.8h, #15
@@ -727,13 +706,12 @@ define void @test_masked_store_success_invert_mask_v4i32(<4 x i32> %x, ptr %ptr,
define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_invert_mask_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -751,35 +729,32 @@ define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr,
define void @test_masked_store_success_invert_mask_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_invert_mask_v16i32:
; SVE: // %bb.0:
-; SVE-NEXT: mov d5, v4.d[1]
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v6.8b, v4.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll2 v5.8h, v4.16b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
+; SVE-NEXT: mov x8, #12 // =0xc
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
-; SVE-NEXT: zip1 v7.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
-; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: shl v7.4s, v7.4s, #31
; SVE-NEXT: shl v4.4s, v4.4s, #31
; SVE-NEXT: cmpge p1.s, p0/z, z6.s, #0
-; SVE-NEXT: shl v7.4s, v7.4s, #31
-; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: cmpge p2.s, p0/z, z5.s, #0
+; SVE-NEXT: cmpge p3.s, p0/z, z7.s, #0
; SVE-NEXT: cmpge p4.s, p0/z, z4.s, #0
-; SVE-NEXT: cmpge p2.s, p0/z, z7.s, #0
-; SVE-NEXT: cmpge p3.s, p0/z, z5.s, #0
-; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
; SVE-NEXT: mov x8, #8 // =0x8
-; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: st1w { z2.s }, p2, [x0, x8, lsl #2]
-; SVE-NEXT: mov x8, #12 // =0xc
-; SVE-NEXT: st1w { z3.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: st1w { z1.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: ret
%load = load <16 x i32>, ptr %ptr, align 32
%sel = select <16 x i1> %mask, <16 x i32> %load, <16 x i32> %x
@@ -814,18 +789,17 @@ define void @test_masked_store_zextload(<4 x i64> %x, ptr %ptr, <4 x i1> %mask)
define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_volatile_load:
; SVE: // %bb.0:
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: ldr q4, [x0]
-; SVE-NEXT: ldr q5, [x0, #16]
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
+; SVE-NEXT: ldr q4, [x0, #16]
+; SVE-NEXT: ldr q5, [x0]
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmlt v3.4s, v3.4s, #0
; SVE-NEXT: cmlt v2.4s, v2.4s, #0
-; SVE-NEXT: bif v0.16b, v4.16b, v3.16b
-; SVE-NEXT: bif v1.16b, v5.16b, v2.16b
+; SVE-NEXT: bif v1.16b, v4.16b, v3.16b
+; SVE-NEXT: bif v0.16b, v5.16b, v2.16b
; SVE-NEXT: stp q0, q1, [x0]
; SVE-NEXT: ret
%load = load volatile <8 x i32>, ptr %ptr, align 32
@@ -837,19 +811,18 @@ define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_volatile_store(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_volatile_store:
; SVE: // %bb.0:
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: ldp q4, q5, [x0]
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
+; SVE-NEXT: ldp q5, q4, [x0]
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmlt v3.4s, v3.4s, #0
; SVE-NEXT: cmlt v2.4s, v2.4s, #0
-; SVE-NEXT: bif v0.16b, v4.16b, v3.16b
-; SVE-NEXT: bif v1.16b, v5.16b, v2.16b
-; SVE-NEXT: str q0, [x0]
+; SVE-NEXT: bif v1.16b, v4.16b, v3.16b
+; SVE-NEXT: bif v0.16b, v5.16b, v2.16b
; SVE-NEXT: str q1, [x0, #16]
+; SVE-NEXT: str q0, [x0]
; SVE-NEXT: ret
%load = load <8 x i32>, ptr %ptr, align 32
%sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load
@@ -874,21 +847,20 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
; SVE-NEXT: movi v1.2d, #0000000000000000
; SVE-NEXT: stp q0, q0, [x0]
; SVE-NEXT: bl use_vec
-; SVE-NEXT: zip2 v0.8b, v8.8b, v0.8b
+; SVE-NEXT: ushll v0.8h, v8.8b, #0
; SVE-NEXT: ldp q3, q2, [sp, #16] // 32-byte Folded Reload
-; SVE-NEXT: zip1 v1.8b, v8.8b, v0.8b
-; SVE-NEXT: ushll v0.4s, v0.4h, #0
; SVE-NEXT: ldr d8, [sp, #64] // 8-byte Reload
+; SVE-NEXT: ushll2 v1.4s, v0.8h, #0
+; SVE-NEXT: ushll v0.4s, v0.4h, #0
+; SVE-NEXT: shl v1.4s, v1.4s, #31
; SVE-NEXT: shl v0.4s, v0.4s, #31
-; SVE-NEXT: ushll v1.4s, v1.4h, #0
+; SVE-NEXT: cmlt v1.4s, v1.4s, #0
; SVE-NEXT: cmlt v0.4s, v0.4s, #0
-; SVE-NEXT: shl v1.4s, v1.4s, #31
-; SVE-NEXT: bsl v0.16b, v2.16b, v3.16b
+; SVE-NEXT: bsl v1.16b, v2.16b, v3.16b
; SVE-NEXT: ldr q2, [sp, #48] // 16-byte Reload
; SVE-NEXT: ldr q3, [sp] // 16-byte Reload
-; SVE-NEXT: cmlt v1.4s, v1.4s, #0
-; SVE-NEXT: bsl v1.16b, v2.16b, v3.16b
-; SVE-NEXT: stp q1, q0, [x19]
+; SVE-NEXT: bsl v0.16b, v2.16b, v3.16b
+; SVE-NEXT: stp q0, q1, [x19]
; SVE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload
; SVE-NEXT: add sp, sp, #96
; SVE-NEXT: ret
@@ -905,16 +877,14 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {
; SVE-LABEL: test_masked_store_multiple_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip1 v6.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v7.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll v5.8h, v5.8b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v5.4s, v5.4h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
@@ -924,10 +894,10 @@ define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %p
; SVE-NEXT: cmlt v6.4s, v6.4s, #0
; SVE-NEXT: cmpne p1.s, p0/z, z7.s, #0
; SVE-NEXT: cmlt v5.4s, v5.4s, #0
-; SVE-NEXT: ldp q7, q16, [x1]
+; SVE-NEXT: ldp q16, q7, [x1]
; SVE-NEXT: cmpne p2.s, p0/z, z4.s, #0
-; SVE-NEXT: bif v2.16b, v7.16b, v6.16b
-; SVE-NEXT: bif v3.16b, v16.16b, v5.16b
+; SVE-NEXT: bif v3.16b, v7.16b, v6.16b
+; SVE-NEXT: bif v2.16b, v16.16b, v5.16b
; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
; SVE-NEXT: st1w { z0.s }, p2, [x0]
; SVE-NEXT: stp q2, q3, [x1]
@@ -946,68 +916,58 @@ define void @test_masked_store_multiple_v8i64(<8 x i64> %x, <8 x i64> %y, ptr %p
; SVE: // %bb.0:
; SVE-NEXT: ldp d16, d17, [sp]
; SVE-NEXT: ptrue p1.d, vl2
-; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: mov x9, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: mov x8, #6 // =0x6
+; SVE-NEXT: mov x8, #2 // =0x2
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov b18, v16.b[0]
-; SVE-NEXT: mov b19, v16.b[4]
-; SVE-NEXT: mov b20, v17.b[4]
-; SVE-NEXT: mov b21, v16.b[6]
-; SVE-NEXT: mov b22, v16.b[2]
-; SVE-NEXT: mov b23, v17.b[6]
-; SVE-NEXT: mov b24, v17.b[2]
-; SVE-NEXT: mov b25, v17.b[0]
-; SVE-NEXT: mov v18.b[4], v16.b[1]
-; SVE-NEXT: mov v19.b[4], v16.b[5]
-; SVE-NEXT: mov v20.b[4], v17.b[5]
-; SVE-NEXT: mov v21.b[4], v16.b[7]
-; SVE-NEXT: mov v22.b[4], v16.b[3]
-; SVE-NEXT: mov v23.b[4], v17.b[7]
-; SVE-NEXT: mov v24.b[4], v17.b[3]
-; SVE-NEXT: mov v25.b[4], v17.b[1]
+; SVE-NEXT: ushll v16.8h, v16.8b, #0
+; SVE-NEXT: ushll v17.8h, v17.8b, #0
+; SVE-NEXT: ushll v18.4s, v16.4h, #0
+; SVE-NEXT: ushll2 v16.4s, v16.8h, #0
+; SVE-NEXT: ushll2 v19.4s, v17.8h, #0
+; SVE-NEXT: ushll v17.4s, v17.4h, #0
+; SVE-NEXT: ushll2 v21.2d, v16.4s, #0
+; SVE-NEXT: ushll2 v20.2d, v18.4s, #0
+; SVE-NEXT: ushll v16.2d, v16.2s, #0
+; SVE-NEXT: ushll2 v22.2d, v19.4s, #0
+; SVE-NEXT: ushll v19.2d, v19.2s, #0
+; SVE-NEXT: ushll2 v23.2d, v17.4s, #0
+; SVE-NEXT: ushll v17.2d, v17.2s, #0
; SVE-NEXT: ushll v18.2d, v18.2s, #0
-; SVE-NEXT: ushll v17.2d, v19.2s, #0
-; SVE-NEXT: ushll v19.2d, v21.2s, #0
-; SVE-NEXT: ushll v21.2d, v25.2s, #0
-; SVE-NEXT: shl v16.2d, v18.2d, #63
-; SVE-NEXT: ushll v18.2d, v20.2s, #0
-; SVE-NEXT: ushll v20.2d, v23.2s, #0
-; SVE-NEXT: shl v17.2d, v17.2d, #63
-; SVE-NEXT: shl v19.2d, v19.2d, #63
; SVE-NEXT: shl v21.2d, v21.2d, #63
-; SVE-NEXT: cmpne p0.d, p1/z, z16.d, #0
-; SVE-NEXT: ushll v16.2d, v22.2s, #0
-; SVE-NEXT: shl v18.2d, v18.2d, #63
-; SVE-NEXT: ushll v22.2d, v24.2s, #0
; SVE-NEXT: shl v20.2d, v20.2d, #63
-; SVE-NEXT: cmpne p2.d, p1/z, z17.d, #0
-; SVE-NEXT: cmpne p3.d, p1/z, z19.d, #0
; SVE-NEXT: shl v16.2d, v16.2d, #63
-; SVE-NEXT: cmlt v17.2d, v18.2d, #0
-; SVE-NEXT: shl v18.2d, v22.2d, #63
-; SVE-NEXT: ldp q19, q22, [x1, #32]
+; SVE-NEXT: shl v19.2d, v19.2d, #63
+; SVE-NEXT: shl v23.2d, v23.2d, #63
+; SVE-NEXT: shl v22.2d, v22.2d, #63
+; SVE-NEXT: shl v17.2d, v17.2d, #63
+; SVE-NEXT: shl v18.2d, v18.2d, #63
+; SVE-NEXT: cmpne p2.d, p1/z, z21.d, #0
+; SVE-NEXT: cmpne p0.d, p1/z, z20.d, #0
; SVE-NEXT: cmpne p4.d, p1/z, z16.d, #0
-; SVE-NEXT: cmlt v16.2d, v20.2d, #0
-; SVE-NEXT: cmlt v20.2d, v21.2d, #0
-; SVE-NEXT: cmlt v18.2d, v18.2d, #0
-; SVE-NEXT: bif v6.16b, v19.16b, v17.16b
-; SVE-NEXT: ldp q17, q19, [x1]
-; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
-; SVE-NEXT: mov x9, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p3, [x0, x8, lsl #3]
-; SVE-NEXT: mov v2.16b, v16.16b
-; SVE-NEXT: mov v3.16b, v20.16b
-; SVE-NEXT: st1d { z1.d }, p4, [x0, x9, lsl #3]
-; SVE-NEXT: mov v1.16b, v18.16b
-; SVE-NEXT: st1d { z0.d }, p0, [x0]
-; SVE-NEXT: bsl v2.16b, v7.16b, v22.16b
-; SVE-NEXT: bsl v3.16b, v4.16b, v17.16b
-; SVE-NEXT: bsl v1.16b, v5.16b, v19.16b
-; SVE-NEXT: stp q6, q2, [x1, #32]
-; SVE-NEXT: stp q3, q1, [x1]
+; SVE-NEXT: cmlt v16.2d, v19.2d, #0
+; SVE-NEXT: cmlt v23.2d, v23.2d, #0
+; SVE-NEXT: cmlt v22.2d, v22.2d, #0
+; SVE-NEXT: cmlt v17.2d, v17.2d, #0
+; SVE-NEXT: ldp q21, q20, [x1, #32]
+; SVE-NEXT: cmpne p3.d, p1/z, z18.d, #0
+; SVE-NEXT: ldp q19, q18, [x1]
+; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: mov v3.16b, v16.16b
+; SVE-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3]
+; SVE-NEXT: mov v1.16b, v23.16b
+; SVE-NEXT: bif v7.16b, v20.16b, v22.16b
+; SVE-NEXT: st1d { z2.d }, p4, [x0, x9, lsl #3]
+; SVE-NEXT: mov v2.16b, v17.16b
+; SVE-NEXT: bsl v3.16b, v6.16b, v21.16b
+; SVE-NEXT: st1d { z0.d }, p3, [x0]
+; SVE-NEXT: bsl v1.16b, v5.16b, v18.16b
+; SVE-NEXT: bsl v2.16b, v4.16b, v19.16b
+; SVE-NEXT: stp q3, q7, [x1, #32]
+; SVE-NEXT: stp q2, q1, [x1]
; SVE-NEXT: ret
%load = load <8 x i64>, ptr %ptr1, align 32
%load2 = load <8 x i64>, ptr %ptr2, align 32
@@ -1066,21 +1026,20 @@ define void @test_masked_store_unaligned_v4i64(<4 x i64> %data, ptr %ptr, <4 x i
define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_unaligned_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: add x8, x0, #1
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: add x9, x0, #17
+; SVE-NEXT: add x8, x0, #17
+; SVE-NEXT: add x9, x0, #1
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmpne p1.s, p0/z, z3.s, #0
; SVE-NEXT: cmpne p2.s, p0/z, z2.s, #0
-; SVE-NEXT: st1w { z0.s }, p1, [x8]
-; SVE-NEXT: st1w { z1.s }, p2, [x9]
+; SVE-NEXT: st1w { z1.s }, p1, [x8]
+; SVE-NEXT: st1w { z0.s }, p2, [x9]
; SVE-NEXT: ret
%ptr_i8 = getelementptr i8, ptr %ptr, i32 1
%ptr_vec = bitcast ptr %ptr_i8 to ptr
@@ -1093,40 +1052,34 @@ define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i
define void @test_masked_store_unaligned_v8i64(<8 x i64> %data, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_unaligned_v8i64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: add x8, x0, #33
-; SVE-NEXT: mov b7, v4.b[0]
-; SVE-NEXT: mov b16, v4.b[2]
-; SVE-NEXT: add x9, x0, #49
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: add x8, x0, #49
+; SVE-NEXT: add x9, x0, #33
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[1]
-; SVE-NEXT: mov v16.b[4], v4.b[3]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x8]
-; SVE-NEXT: add x8, x0, #1
-; SVE-NEXT: st1d { z3.d }, p2, [x9]
-; SVE-NEXT: add x9, x0, #17
-; SVE-NEXT: st1d { z0.d }, p3, [x8]
-; SVE-NEXT: st1d { z1.d }, p4, [x9]
+; SVE-NEXT: st1d { z3.d }, p1, [x8]
+; SVE-NEXT: add x8, x0, #17
+; SVE-NEXT: st1d { z2.d }, p2, [x9]
+; SVE-NEXT: add x9, x0, #1
+; SVE-NEXT: st1d { z1.d }, p3, [x8]
+; SVE-NEXT: st1d { z0.d }, p4, [x9]
; SVE-NEXT: ret
%ptr_i8 = getelementptr i8, ptr %ptr, i64 1
%ptr_vec = bitcast ptr %ptr_i8 to ptr
diff --git a/llvm/test/CodeGen/AArch64/neon-extadd.ll b/llvm/test/CodeGen/AArch64/neon-extadd.ll
index 627dab6402791..b48fb576ff1e0 100644
--- a/llvm/test/CodeGen/AArch64/neon-extadd.ll
+++ b/llvm/test/CodeGen/AArch64/neon-extadd.ll
@@ -1368,95 +1368,69 @@ entry:
define <16 x i32> @i12(<16 x i12> %s0, <16 x i12> %s1) {
; CHECK-SD-LABEL: i12:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: str x23, [sp, #-48]! // 8-byte Folded Spill
-; CHECK-SD-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT: .cfi_offset w19, -8
-; CHECK-SD-NEXT: .cfi_offset w20, -16
-; CHECK-SD-NEXT: .cfi_offset w21, -24
-; CHECK-SD-NEXT: .cfi_offset w22, -32
-; CHECK-SD-NEXT: .cfi_offset w23, -48
-; CHECK-SD-NEXT: ldr w13, [sp, #112]
-; CHECK-SD-NEXT: ldr w14, [sp, #144]
-; CHECK-SD-NEXT: fmov s2, w4
-; CHECK-SD-NEXT: ldr w17, [sp, #176]
-; CHECK-SD-NEXT: ldr w19, [sp, #208]
-; CHECK-SD-NEXT: fmov s3, w0
-; CHECK-SD-NEXT: ldr w20, [sp, #80]
-; CHECK-SD-NEXT: ldr w21, [sp, #48]
-; CHECK-SD-NEXT: fmov s5, w13
-; CHECK-SD-NEXT: fmov s4, w19
-; CHECK-SD-NEXT: fmov s6, w17
-; CHECK-SD-NEXT: fmov s7, w14
-; CHECK-SD-NEXT: fmov s0, w20
-; CHECK-SD-NEXT: fmov s1, w21
-; CHECK-SD-NEXT: ldr w10, [sp, #120]
-; CHECK-SD-NEXT: ldr w11, [sp, #152]
-; CHECK-SD-NEXT: ldr w12, [sp, #184]
-; CHECK-SD-NEXT: ldr w15, [sp, #216]
-; CHECK-SD-NEXT: ldr w22, [sp, #88]
-; CHECK-SD-NEXT: ldr w23, [sp, #56]
-; CHECK-SD-NEXT: mov v2.h[1], w5
-; CHECK-SD-NEXT: mov v3.h[1], w1
-; CHECK-SD-NEXT: mov v5.h[1], w10
-; CHECK-SD-NEXT: mov v4.h[1], w15
-; CHECK-SD-NEXT: mov v0.h[1], w22
-; CHECK-SD-NEXT: mov v1.h[1], w23
-; CHECK-SD-NEXT: mov v6.h[1], w12
-; CHECK-SD-NEXT: mov v7.h[1], w11
-; CHECK-SD-NEXT: ldr w8, [sp, #128]
-; CHECK-SD-NEXT: ldr w9, [sp, #160]
-; CHECK-SD-NEXT: ldr w16, [sp, #64]
-; CHECK-SD-NEXT: ldr w18, [sp, #96]
-; CHECK-SD-NEXT: ldr w10, [sp, #192]
-; CHECK-SD-NEXT: ldr w11, [sp, #224]
-; CHECK-SD-NEXT: mov v2.h[2], w6
-; CHECK-SD-NEXT: mov v3.h[2], w2
-; CHECK-SD-NEXT: mov v0.h[2], w18
-; CHECK-SD-NEXT: mov v1.h[2], w16
-; CHECK-SD-NEXT: mov v5.h[2], w8
-; CHECK-SD-NEXT: mov v4.h[2], w11
-; CHECK-SD-NEXT: mov v6.h[2], w10
-; CHECK-SD-NEXT: mov v7.h[2], w9
-; CHECK-SD-NEXT: ldr w12, [sp, #72]
-; CHECK-SD-NEXT: ldr w13, [sp, #104]
-; CHECK-SD-NEXT: ldr w8, [sp, #136]
-; CHECK-SD-NEXT: ldr w9, [sp, #168]
-; CHECK-SD-NEXT: ldr w10, [sp, #200]
-; CHECK-SD-NEXT: ldr w11, [sp, #232]
-; CHECK-SD-NEXT: mov v0.h[3], w13
-; CHECK-SD-NEXT: mov v1.h[3], w12
-; CHECK-SD-NEXT: mov v2.h[3], w7
-; CHECK-SD-NEXT: mov v3.h[3], w3
-; CHECK-SD-NEXT: mov v5.h[3], w8
-; CHECK-SD-NEXT: mov v4.h[3], w11
-; CHECK-SD-NEXT: mov v6.h[3], w10
-; CHECK-SD-NEXT: mov v7.h[3], w9
-; CHECK-SD-NEXT: movi v16.4s, #15, msl #8
-; CHECK-SD-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-SD-NEXT: ushll v4.4s, v4.4h, #0
-; CHECK-SD-NEXT: ushll v6.4s, v6.4h, #0
-; CHECK-SD-NEXT: ushll v7.4s, v7.4h, #0
-; CHECK-SD-NEXT: and v17.16b, v0.16b, v16.16b
-; CHECK-SD-NEXT: and v18.16b, v1.16b, v16.16b
-; CHECK-SD-NEXT: and v1.16b, v2.16b, v16.16b
-; CHECK-SD-NEXT: and v0.16b, v3.16b, v16.16b
-; CHECK-SD-NEXT: and v2.16b, v5.16b, v16.16b
-; CHECK-SD-NEXT: and v3.16b, v4.16b, v16.16b
-; CHECK-SD-NEXT: and v4.16b, v6.16b, v16.16b
-; CHECK-SD-NEXT: and v5.16b, v7.16b, v16.16b
-; CHECK-SD-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-SD-NEXT: add v3.4s, v17.4s, v3.4s
-; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s
-; CHECK-SD-NEXT: add v2.4s, v18.4s, v4.4s
-; CHECK-SD-NEXT: ldr x23, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT: ldr s2, [sp, #32]
+; CHECK-SD-NEXT: add x8, sp, #40
+; CHECK-SD-NEXT: fmov s0, w4
+; CHECK-SD-NEXT: fmov s1, w0
+; CHECK-SD-NEXT: ldr s4, [sp]
+; CHECK-SD-NEXT: add x9, sp, #8
+; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #48
+; CHECK-SD-NEXT: ldr s6, [sp, #64]
+; CHECK-SD-NEXT: ld1 { v4.s }[1], [x9]
+; CHECK-SD-NEXT: mov v0.s[1], w5
+; CHECK-SD-NEXT: add x9, sp, #16
+; CHECK-SD-NEXT: mov v1.s[1], w1
+; CHECK-SD-NEXT: ldr s5, [sp, #96]
+; CHECK-SD-NEXT: add x11, sp, #72
+; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #56
+; CHECK-SD-NEXT: ldr s7, [sp, #160]
+; CHECK-SD-NEXT: ldr s16, [sp, #128]
+; CHECK-SD-NEXT: add x10, sp, #104
+; CHECK-SD-NEXT: ld1 { v4.s }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v6.s }[1], [x11]
+; CHECK-SD-NEXT: add x9, sp, #136
+; CHECK-SD-NEXT: ld1 { v5.s }[1], [x10]
+; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #168
+; CHECK-SD-NEXT: ld1 { v16.s }[1], [x9]
+; CHECK-SD-NEXT: ld1 { v7.s }[1], [x8]
+; CHECK-SD-NEXT: mov v0.s[2], w6
+; CHECK-SD-NEXT: mov v1.s[2], w2
+; CHECK-SD-NEXT: add x10, sp, #80
+; CHECK-SD-NEXT: add x9, sp, #112
+; CHECK-SD-NEXT: add x11, sp, #144
+; CHECK-SD-NEXT: ld1 { v6.s }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #176
+; CHECK-SD-NEXT: ld1 { v5.s }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v7.s }[2], [x10]
+; CHECK-SD-NEXT: ld1 { v16.s }[2], [x11]
+; CHECK-SD-NEXT: add x8, sp, #24
+; CHECK-SD-NEXT: add x9, sp, #88
+; CHECK-SD-NEXT: movi v3.4s, #15, msl #8
+; CHECK-SD-NEXT: mov v0.s[3], w7
+; CHECK-SD-NEXT: mov v1.s[3], w3
+; CHECK-SD-NEXT: ld1 { v4.s }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #120
+; CHECK-SD-NEXT: ld1 { v6.s }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #184
+; CHECK-SD-NEXT: add x10, sp, #152
+; CHECK-SD-NEXT: ld1 { v5.s }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v7.s }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v16.s }[3], [x10]
+; CHECK-SD-NEXT: and v17.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT: and v0.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT: and v1.16b, v2.16b, v3.16b
+; CHECK-SD-NEXT: and v2.16b, v4.16b, v3.16b
+; CHECK-SD-NEXT: and v4.16b, v6.16b, v3.16b
+; CHECK-SD-NEXT: and v5.16b, v5.16b, v3.16b
+; CHECK-SD-NEXT: and v6.16b, v7.16b, v3.16b
+; CHECK-SD-NEXT: and v3.16b, v16.16b, v3.16b
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-SD-NEXT: add v2.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT: add v3.4s, v1.4s, v6.4s
+; CHECK-SD-NEXT: add v1.4s, v17.4s, v5.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: i12:
diff --git a/llvm/test/CodeGen/AArch64/sext.ll b/llvm/test/CodeGen/AArch64/sext.ll
index 278731ad0663a..f1fae4aafd26a 100644
--- a/llvm/test/CodeGen/AArch64/sext.ll
+++ b/llvm/test/CodeGen/AArch64/sext.ll
@@ -239,9 +239,8 @@ define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) {
; CHECK-SD-LABEL: sext_v3i8_v3i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v0.s[2], w2
; CHECK-SD-NEXT: shl v0.4s, v0.4s, #24
; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #24
; CHECK-SD-NEXT: ret
@@ -398,9 +397,8 @@ define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) {
; CHECK-SD-LABEL: sext_v3i10_v3i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v0.s[2], w2
; CHECK-SD-NEXT: shl v0.4s, v0.4s, #22
; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #22
; CHECK-SD-NEXT: ret
@@ -1055,42 +1053,36 @@ entry:
define <16 x i32> @sext_v16i10_v16i32(<16 x i10> %a) {
; CHECK-SD-LABEL: sext_v16i10_v16i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ldr w8, [sp, #32]
-; CHECK-SD-NEXT: ldr w9, [sp]
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: fmov s1, w4
-; CHECK-SD-NEXT: ldr w10, [sp, #40]
-; CHECK-SD-NEXT: ldr w11, [sp, #8]
-; CHECK-SD-NEXT: fmov s2, w9
-; CHECK-SD-NEXT: fmov s3, w8
-; CHECK-SD-NEXT: ldr w8, [sp, #48]
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: ldr w9, [sp, #16]
-; CHECK-SD-NEXT: mov v1.h[1], w5
-; CHECK-SD-NEXT: mov v2.h[1], w11
-; CHECK-SD-NEXT: mov v3.h[1], w10
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: mov v1.h[2], w6
-; CHECK-SD-NEXT: mov v2.h[2], w9
-; CHECK-SD-NEXT: mov v3.h[2], w8
-; CHECK-SD-NEXT: ldr w8, [sp, #56]
-; CHECK-SD-NEXT: ldr w9, [sp, #24]
-; CHECK-SD-NEXT: mov v0.h[3], w3
-; CHECK-SD-NEXT: mov v1.h[3], w7
-; CHECK-SD-NEXT: mov v2.h[3], w9
-; CHECK-SD-NEXT: mov v3.h[3], w8
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT: shl v0.4s, v0.4s, #22
-; CHECK-SD-NEXT: shl v1.4s, v1.4s, #22
+; CHECK-SD-NEXT: ldr s2, [sp]
+; CHECK-SD-NEXT: ldr s3, [sp, #32]
+; CHECK-SD-NEXT: add x8, sp, #8
+; CHECK-SD-NEXT: add x9, sp, #40
+; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #16
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v1.s[1], w5
+; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #48
+; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #24
+; CHECK-SD-NEXT: ld1 { v3.s }[2], [x9]
+; CHECK-SD-NEXT: add x9, sp, #56
+; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: mov v1.s[2], w6
+; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v3.s }[3], [x9]
+; CHECK-SD-NEXT: mov v0.s[3], w3
+; CHECK-SD-NEXT: mov v1.s[3], w7
; CHECK-SD-NEXT: shl v2.4s, v2.4s, #22
; CHECK-SD-NEXT: shl v3.4s, v3.4s, #22
-; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #22
-; CHECK-SD-NEXT: sshr v1.4s, v1.4s, #22
; CHECK-SD-NEXT: sshr v2.4s, v2.4s, #22
+; CHECK-SD-NEXT: shl v0.4s, v0.4s, #22
+; CHECK-SD-NEXT: shl v1.4s, v1.4s, #22
; CHECK-SD-NEXT: sshr v3.4s, v3.4s, #22
+; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #22
+; CHECK-SD-NEXT: sshr v1.4s, v1.4s, #22
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: sext_v16i10_v16i32:
diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
index 85af93eeadbdf..0b6e1dc7c0bbb 100644
--- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
@@ -40,12 +40,14 @@ define <4 x i32> @load_v3i8_to_4xi32(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #2]
; CHECK-NEXT: ldrh w9, [x0]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32:
@@ -53,16 +55,16 @@ define <4 x i32> @load_v3i8_to_4xi32(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -78,12 +80,14 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #2]
; CHECK-NEXT: ldrh w9, [x0]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_align_2:
@@ -91,16 +95,16 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -114,21 +118,25 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
define <4 x i32> @load_v3i8_to_4xi32_align_4(ptr %src) {
; CHECK-LABEL: load_v3i8_to_4xi32_align_4:
; CHECK: ; %bb.0:
-; CHECK-NEXT: ldr s0, [x0]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: ldr s1, [x0]
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_align_4:
; BE: // %bb.0:
-; BE-NEXT: ldr s0, [x0]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: ldr s1, [x0]
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
+; BE-NEXT: rev32 v1.16b, v1.16b
+; BE-NEXT: umov w8, v1.b[1]
+; BE-NEXT: umov w9, v1.b[2]
+; BE-NEXT: mov v1.s[1], w8
+; BE-NEXT: mov v1.s[2], w9
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: ret
@@ -143,12 +151,14 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_1(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #3]
; CHECK-NEXT: ldurh w9, [x0, #1]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_const_offset_1:
@@ -156,16 +166,16 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_1(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldurh w8, [x0, #1]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #3]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -182,12 +192,14 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_3(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #5]
; CHECK-NEXT: ldurh w9, [x0, #3]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_const_offset_3:
@@ -195,16 +207,16 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_3(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldurh w8, [x0, #3]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #5]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -222,15 +234,15 @@ define <4 x i32> @volatile_load_v3i8_to_4xi32(ptr %src) {
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: ldrh w8, [x0]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: strh w8, [sp, #12]
-; CHECK-NEXT: ldr s0, [sp, #12]
+; CHECK-NEXT: ldr s1, [sp, #12]
+; CHECK-NEXT: ushll.8h v1, v1, #0
+; CHECK-NEXT: umov.h w8, v1[1]
+; CHECK-NEXT: mov.s v1[1], w8
; CHECK-NEXT: ldrsb w8, [x0, #2]
-; CHECK-NEXT: ushll.8h v0, v0, #0
-; CHECK-NEXT: mov.h v0[1], v0[1]
-; CHECK-NEXT: mov.h v0[2], w8
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: mov.s v1[2], w8
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: ret
;
@@ -239,16 +251,16 @@ define <4 x i32> @volatile_load_v3i8_to_4xi32(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
index df78c2394a942..a1766dfef9465 100644
--- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
@@ -141,23 +141,20 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, ptr %p2) nounwind {
; CHECK-NEXT: add v4.16b, v0.16b, v1.16b
; CHECK-NEXT: cmhi v0.16b, v0.16b, v4.16b
; CHECK-NEXT: str q4, [x0]
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-NEXT: shl v1.4s, v1.4s, #31
; CHECK-NEXT: shl v5.4s, v0.4s, #31
; CHECK-NEXT: cmlt v0.4s, v2.4s, #0
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: shl v6.4s, v1.4s, #31
-; CHECK-NEXT: cmlt v1.4s, v5.4s, #0
-; CHECK-NEXT: cmlt v2.4s, v3.4s, #0
-; CHECK-NEXT: cmlt v3.4s, v6.4s, #0
+; CHECK-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
; CHECK-NEXT: ret
%t = call {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
%val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
@@ -173,15 +170,8 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK-NEXT: add v2.8h, v0.8h, v1.8h
; CHECK-NEXT: cmhi v0.8h, v0.8h, v2.8h
; CHECK-NEXT: str q2, [x0]
-; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NEXT: shl v1.4s, v1.4s, #31
-; CHECK-NEXT: shl v3.4s, v0.4s, #31
-; CHECK-NEXT: cmlt v0.4s, v1.4s, #0
-; CHECK-NEXT: cmlt v1.4s, v3.4s, #0
+; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vec_umulo.ll b/llvm/test/CodeGen/AArch64/vec_umulo.ll
index 0a3d94d8ba952..66757b6f7cf51 100644
--- a/llvm/test/CodeGen/AArch64/vec_umulo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_umulo.ll
@@ -165,27 +165,24 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, ptr %p2) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: umull2 v2.8h, v0.16b, v1.16b
; CHECK-NEXT: umull v3.8h, v0.8b, v1.8b
-; CHECK-NEXT: mul v6.16b, v0.16b, v1.16b
+; CHECK-NEXT: mul v7.16b, v0.16b, v1.16b
; CHECK-NEXT: uzp2 v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: str q6, [x0]
+; CHECK-NEXT: str q7, [x0]
; CHECK-NEXT: cmtst v2.16b, v2.16b, v2.16b
-; CHECK-NEXT: mov d3, v2.d[1]
-; CHECK-NEXT: zip1 v4.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip1 v5.8b, v3.8b, v0.8b
-; CHECK-NEXT: zip2 v3.8b, v3.8b, v0.8b
-; CHECK-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-NEXT: ushll v3.8h, v2.8b, #0
+; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEXT: ushll v4.4s, v3.4h, #0
+; CHECK-NEXT: ushll2 v5.4s, v2.8h, #0
+; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0
; CHECK-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-NEXT: shl v4.4s, v4.4s, #31
-; CHECK-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: shl v5.4s, v5.4s, #31
+; CHECK-NEXT: shl v6.4s, v3.4s, #31
; CHECK-NEXT: shl v2.4s, v2.4s, #31
; CHECK-NEXT: cmlt v0.4s, v4.4s, #0
-; CHECK-NEXT: shl v5.4s, v5.4s, #31
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: cmlt v1.4s, v2.4s, #0
-; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
-; CHECK-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v3.4s, v5.4s, #0
+; CHECK-NEXT: cmlt v1.4s, v6.4s, #0
+; CHECK-NEXT: cmlt v2.4s, v2.4s, #0
; CHECK-NEXT: ret
%t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
%val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
@@ -200,19 +197,12 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
; CHECK-NEXT: umull v3.4s, v0.4h, v1.4h
-; CHECK-NEXT: mul v4.8h, v0.8h, v1.8h
; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: str q4, [x0]
+; CHECK-NEXT: mul v3.8h, v0.8h, v1.8h
; CHECK-NEXT: cmtst v2.8h, v2.8h, v2.8h
-; CHECK-NEXT: xtn v2.8b, v2.8h
-; CHECK-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: cmlt v0.4s, v3.4s, #0
-; CHECK-NEXT: cmlt v1.4s, v2.4s, #0
+; CHECK-NEXT: str q3, [x0]
+; CHECK-NEXT: sshll2 v1.4s, v2.8h, #0
+; CHECK-NEXT: sshll v0.4s, v2.4h, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vector-compress.ll b/llvm/test/CodeGen/AArch64/vector-compress.ll
index 55c343164a1b8..74f38884fe9f6 100644
--- a/llvm/test/CodeGen/AArch64/vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/vector-compress.ll
@@ -413,10 +413,9 @@ define <3 x i32> @test_compress_narrow(<3 x i32> %vec, <3 x i1> %mask) {
; CHECK-NEXT: movi.2d v1, #0000000000000000
; CHECK-NEXT: mov x11, sp
; CHECK-NEXT: str s0, [sp]
-; CHECK-NEXT: mov.h v1[0], w0
-; CHECK-NEXT: mov.h v1[1], w1
-; CHECK-NEXT: mov.h v1[2], w2
-; CHECK-NEXT: ushll.4s v1, v1, #0
+; CHECK-NEXT: mov.s v1[0], w0
+; CHECK-NEXT: mov.s v1[1], w1
+; CHECK-NEXT: mov.s v1[2], w2
; CHECK-NEXT: shl.4s v1, v1, #31
; CHECK-NEXT: cmlt.4s v1, v1, #0
; CHECK-NEXT: mov.s w8, v1[1]
diff --git a/llvm/test/CodeGen/AArch64/vselect-ext.ll b/llvm/test/CodeGen/AArch64/vselect-ext.ll
index 5ed427110ea2f..5e11c868b549b 100644
--- a/llvm/test/CodeGen/AArch64/vselect-ext.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-ext.ll
@@ -222,14 +222,16 @@ define <3 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v3i16(<3 x i8>
; CHECK-NEXT: movi.2d v3, #0x0000ff000000ff
; CHECK-NEXT: Lloh1:
; CHECK-NEXT: ldr d2, [x8, lCPI9_0 at PAGEOFF]
-; CHECK-NEXT: mov.h v0[1], w1
-; CHECK-NEXT: mov.h v0[2], w2
-; CHECK-NEXT: ushll.4s v1, v0, #0
-; CHECK-NEXT: bic.4h v0, #255, lsl #8
-; CHECK-NEXT: cmhi.4h v0, v0, v2
-; CHECK-NEXT: and.16b v1, v1, v3
-; CHECK-NEXT: sshll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v1, v0
+; CHECK-NEXT: mov.16b v1, v0
+; CHECK-NEXT: mov.s v0[1], w1
+; CHECK-NEXT: mov.h v1[1], w1
+; CHECK-NEXT: mov.s v0[2], w2
+; CHECK-NEXT: mov.h v1[2], w2
+; CHECK-NEXT: and.16b v0, v0, v3
+; CHECK-NEXT: bic.4h v1, #255, lsl #8
+; CHECK-NEXT: cmhi.4h v1, v1, v2
+; CHECK-NEXT: sshll.4s v1, v1, #0
+; CHECK-NEXT: and.16b v0, v0, v1
; CHECK-NEXT: ret
; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh1
%ext = zext <3 x i8> %a to <3 x i32>
diff --git a/llvm/test/CodeGen/AArch64/zext.ll b/llvm/test/CodeGen/AArch64/zext.ll
index 9ec760aa57e27..8f02da9c78223 100644
--- a/llvm/test/CodeGen/AArch64/zext.ll
+++ b/llvm/test/CodeGen/AArch64/zext.ll
@@ -260,9 +260,8 @@ define <3 x i32> @zext_v3i8_v3i32(<3 x i8> %a) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: movi v1.2d, #0x0000ff000000ff
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v0.s[2], w2
; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT: ret
;
@@ -416,9 +415,8 @@ define <3 x i32> @zext_v3i10_v3i32(<3 x i10> %a) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: movi v1.4s, #3, msl #8
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v0.s[2], w2
; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT: ret
;
@@ -1044,39 +1042,33 @@ entry:
define <16 x i32> @zext_v16i10_v16i32(<16 x i10> %a) {
; CHECK-SD-LABEL: zext_v16i10_v16i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ldr w8, [sp, #32]
-; CHECK-SD-NEXT: ldr w9, [sp]
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: fmov s1, w4
-; CHECK-SD-NEXT: ldr w10, [sp, #40]
-; CHECK-SD-NEXT: ldr w11, [sp, #8]
-; CHECK-SD-NEXT: fmov s2, w9
-; CHECK-SD-NEXT: fmov s3, w8
-; CHECK-SD-NEXT: ldr w8, [sp, #48]
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: ldr w9, [sp, #16]
+; CHECK-SD-NEXT: ldr s2, [sp]
+; CHECK-SD-NEXT: ldr s3, [sp, #32]
+; CHECK-SD-NEXT: add x8, sp, #8
+; CHECK-SD-NEXT: add x9, sp, #40
+; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #16
; CHECK-SD-NEXT: movi v4.4s, #3, msl #8
-; CHECK-SD-NEXT: mov v1.h[1], w5
-; CHECK-SD-NEXT: mov v2.h[1], w11
-; CHECK-SD-NEXT: mov v3.h[1], w10
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: mov v1.h[2], w6
-; CHECK-SD-NEXT: mov v2.h[2], w9
-; CHECK-SD-NEXT: mov v3.h[2], w8
-; CHECK-SD-NEXT: ldr w8, [sp, #56]
-; CHECK-SD-NEXT: ldr w9, [sp, #24]
-; CHECK-SD-NEXT: mov v0.h[3], w3
-; CHECK-SD-NEXT: mov v1.h[3], w7
-; CHECK-SD-NEXT: mov v2.h[3], w9
-; CHECK-SD-NEXT: mov v3.h[3], w8
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT: and v0.16b, v0.16b, v4.16b
-; CHECK-SD-NEXT: and v1.16b, v1.16b, v4.16b
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v1.s[1], w5
+; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #48
+; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #24
+; CHECK-SD-NEXT: ld1 { v3.s }[2], [x9]
+; CHECK-SD-NEXT: add x9, sp, #56
+; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: mov v1.s[2], w6
+; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v3.s }[3], [x9]
+; CHECK-SD-NEXT: mov v0.s[3], w3
+; CHECK-SD-NEXT: mov v1.s[3], w7
; CHECK-SD-NEXT: and v2.16b, v2.16b, v4.16b
; CHECK-SD-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-SD-NEXT: and v1.16b, v1.16b, v4.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: zext_v16i10_v16i32:
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index db53dda401bc2..fefd65b26be34 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -4422,22 +4422,18 @@ define <64 x i1> @mask64_insert(i32 %a) {
;
; SKX-LABEL: mask64_insert:
; SKX: ## %bb.0:
-; SKX-NEXT: andb $1, %dil
-; SKX-NEXT: cmpb $1, %dil
-; SKX-NEXT: movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
-; SKX-NEXT: sbbq $-1, %rax
-; SKX-NEXT: kmovq %rax, %k0
-; SKX-NEXT: vpmovm2b %k0, %zmm0
+; SKX-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SKX-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; SKX-NEXT: vpinsrb $0, %edi, %xmm0, %xmm1
+; SKX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; SKX-NEXT: retq
;
; AVX512BW-LABEL: mask64_insert:
; AVX512BW: ## %bb.0:
-; AVX512BW-NEXT: andb $1, %dil
-; AVX512BW-NEXT: cmpb $1, %dil
-; AVX512BW-NEXT: movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
-; AVX512BW-NEXT: sbbq $-1, %rax
-; AVX512BW-NEXT: kmovq %rax, %k0
-; AVX512BW-NEXT: vpmovm2b %k0, %zmm0
+; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512BW-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT: vpinsrb $0, %edi, %xmm0, %xmm1
+; AVX512BW-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: retq
;
; AVX512DQ-LABEL: mask64_insert:
@@ -4454,16 +4450,10 @@ define <64 x i1> @mask64_insert(i32 %a) {
;
; X86-LABEL: mask64_insert:
; X86: ## %bb.0:
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: andb $1, %al
-; X86-NEXT: cmpb $1, %al
-; X86-NEXT: movl $-131076, %eax ## imm = 0xFFFDFFFC
-; X86-NEXT: sbbl $-1, %eax
-; X86-NEXT: kmovd %eax, %k0
-; X86-NEXT: movl $-131075, %eax ## imm = 0xFFFDFFFD
-; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kunpckdq %k0, %k1, %k0
-; X86-NEXT: vpmovm2b %k0, %zmm0
+; X86-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; X86-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X86-NEXT: vpinsrb $0, {{[0-9]+}}(%esp), %xmm0, %xmm1
+; X86-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; X86-NEXT: retl
%a_i = trunc i32 %a to i1
%maskv = insertelement <64 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
More information about the llvm-commits
mailing list