[llvm] [AArch64][SVE] Enable known bits for predicated shifts (PR #200347)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 02:07:12 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/200347
>From 6954a94de4337627bc819ffd6e4b557cb5756946 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Thu, 28 May 2026 10:44:45 +0000
Subject: [PATCH 1/2] [AArch64][SVE] Enable known bits for predicated shifts
Allow SelectionDAG to query target known-bits information for scalable
vector nodes, and known-bits cases for SVE predicated SHL, SRL and SRA
nodes.
This enables DAG combines to prove disjointness for ORs involving scalable
vector shifts, enabling USRA/SSRA instruction selection.
---
.../CodeGen/AArch64/sve2-fixed-length-sra.ll | 2012 +++++++++++++++++
llvm/test/CodeGen/AArch64/sve2-sra.ll | 245 ++
2 files changed, 2257 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll
diff --git a/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll b/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll
new file mode 100644
index 0000000000000..891724998a913
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll
@@ -0,0 +1,2012 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; USRA disjoint OR
+;
+
+define <8 x i8> @usra_disjoint_or8xi8(<8 x i8> %a, <8 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or8xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: usra v0.8b, v1.8b, #4
+; CHECK-NEXT: ret
+ %shift = lshr <8 x i8> %b, splat(i8 4)
+ %res = or disjoint <8 x i8> %a, %shift
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @usra_disjoint_or16xi8(<16 x i8> %a, <16 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or16xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: usra v0.16b, v1.16b, #4
+; CHECK-NEXT: ret
+ %shift = lshr <16 x i8> %b, splat(i8 4)
+ %res = or disjoint <16 x i8> %a, %shift
+ ret <16 x i8> %res
+}
+
+define void @usra_disjoint_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or32xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.b, z0.b, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i8>, ptr %a
+ %vb = load <32 x i8>, ptr %b
+ %shift = lshr <32 x i8> %vb, splat(i8 4)
+ %res = or disjoint <32 x i8> %va, %shift
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or64xi8(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_or64xi8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: lsr z0.b, z0.b, #4
+; VBITS_GE_256-NEXT: lsr z1.b, z1.b, #4
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_or64xi8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.b, vl64
+; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT: lsr z0.b, z0.b, #4
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <64 x i8>, ptr %a
+ %vb = load <64 x i8>, ptr %b
+ %shift = lshr <64 x i8> %vb, splat(i8 4)
+ %res = or disjoint <64 x i8> %va, %shift
+ store <64 x i8> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_or128xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl128
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.b, z0.b, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i8>, ptr %a
+ %vb = load <128 x i8>, ptr %b
+ %shift = lshr <128 x i8> %vb, splat(i8 4)
+ %res = or disjoint <128 x i8> %va, %shift
+ store <128 x i8> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_or256xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl256
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.b, z0.b, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <256 x i8>, ptr %a
+ %vb = load <256 x i8>, ptr %b
+ %shift = lshr <256 x i8> %vb, splat(i8 4)
+ %res = or disjoint <256 x i8> %va, %shift
+ store <256 x i8> %res, ptr %a
+ ret void
+}
+
+define <4 x i16> @usra_disjoint_or4xi16(<4 x i16> %a, <4 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or4xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: usra v0.4h, v1.4h, #4
+; CHECK-NEXT: ret
+ %shift = lshr <4 x i16> %b, splat(i16 4)
+ %res = or disjoint <4 x i16> %a, %shift
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @usra_disjoint_or8xi16(<8 x i16> %a, <8 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: usra v0.8h, v1.8h, #4
+; CHECK-NEXT: ret
+ %shift = lshr <8 x i16> %b, splat(i16 4)
+ %res = or disjoint <8 x i16> %a, %shift
+ ret <8 x i16> %res
+}
+
+define void @usra_disjoint_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or16xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl16
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.h, z0.h, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <16 x i16>, ptr %a
+ %vb = load <16 x i16>, ptr %b
+ %shift = lshr <16 x i16> %vb, splat(i16 4)
+ %res = or disjoint <16 x i16> %va, %shift
+ store <16 x i16> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or32xi16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_or32xi16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: lsr z0.h, z0.h, #4
+; VBITS_GE_256-NEXT: lsr z1.h, z1.h, #4
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_or32xi16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.h, vl32
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: lsr z0.h, z0.h, #4
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <32 x i16>, ptr %a
+ %vb = load <32 x i16>, ptr %b
+ %shift = lshr <32 x i16> %vb, splat(i16 4)
+ %res = or disjoint <32 x i16> %va, %shift
+ store <32 x i16> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_or64xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.h, z0.h, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i16>, ptr %a
+ %vb = load <64 x i16>, ptr %b
+ %shift = lshr <64 x i16> %vb, splat(i16 4)
+ %res = or disjoint <64 x i16> %va, %shift
+ store <64 x i16> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_or128xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.h, z0.h, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i16>, ptr %a
+ %vb = load <128 x i16>, ptr %b
+ %shift = lshr <128 x i16> %vb, splat(i16 4)
+ %res = or disjoint <128 x i16> %va, %shift
+ store <128 x i16> %res, ptr %a
+ ret void
+}
+
+define <2 x i32> @usra_disjoint_or2xi32(<2 x i32> %a, <2 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or2xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: usra v0.2s, v1.2s, #4
+; CHECK-NEXT: ret
+ %shift = lshr <2 x i32> %b, splat(i32 4)
+ %res = or disjoint <2 x i32> %a, %shift
+ ret <2 x i32> %res
+}
+
+define <4 x i32> @usra_disjoint_or4xi32(<4 x i32> %a, <4 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: usra v0.4s, v1.4s, #4
+; CHECK-NEXT: ret
+ %shift = lshr <4 x i32> %b, splat(i32 4)
+ %res = or disjoint <4 x i32> %a, %shift
+ ret <4 x i32> %res
+}
+
+define void @usra_disjoint_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or8xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.s, z0.s, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <8 x i32>, ptr %a
+ %vb = load <8 x i32>, ptr %b
+ %shift = lshr <8 x i32> %vb, splat(i32 4)
+ %res = or disjoint <8 x i32> %va, %shift
+ store <8 x i32> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or16xi32(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_or16xi32:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: lsr z0.s, z0.s, #4
+; VBITS_GE_256-NEXT: lsr z1.s, z1.s, #4
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_or16xi32:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: lsr z0.s, z0.s, #4
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <16 x i32>, ptr %a
+ %vb = load <16 x i32>, ptr %b
+ %shift = lshr <16 x i32> %vb, splat(i32 4)
+ %res = or disjoint <16 x i32> %va, %shift
+ store <16 x i32> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_or32xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.s, z0.s, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i32>, ptr %a
+ %vb = load <32 x i32>, ptr %b
+ %shift = lshr <32 x i32> %vb, splat(i32 4)
+ %res = or disjoint <32 x i32> %va, %shift
+ store <32 x i32> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_or64xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.s, z0.s, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i32>, ptr %a
+ %vb = load <64 x i32>, ptr %b
+ %shift = lshr <64 x i32> %vb, splat(i32 4)
+ %res = or disjoint <64 x i32> %va, %shift
+ store <64 x i32> %res, ptr %a
+ ret void
+}
+
+define <1 x i64> @usra_disjoint_or1xi64(<1 x i64> %a, <1 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or1xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: usra d0, d1, #4
+; CHECK-NEXT: ret
+ %shift = lshr <1 x i64> %b, splat(i64 4)
+ %res = or disjoint <1 x i64> %a, %shift
+ ret <1 x i64> %res
+}
+
+define <2 x i64> @usra_disjoint_or2xi64(<2 x i64> %a, <2 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or2xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: usra v0.2d, v1.2d, #4
+; CHECK-NEXT: ret
+ %shift = lshr <2 x i64> %b, splat(i64 4)
+ %res = or disjoint <2 x i64> %a, %shift
+ ret <2 x i64> %res
+}
+
+define void @usra_disjoint_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or4xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.d, z0.d, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <4 x i64>, ptr %a
+ %vb = load <4 x i64>, ptr %b
+ %shift = lshr <4 x i64> %vb, splat(i64 4)
+ %res = or disjoint <4 x i64> %va, %shift
+ store <4 x i64> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or8xi64(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_or8xi64:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: lsr z0.d, z0.d, #4
+; VBITS_GE_256-NEXT: lsr z1.d, z1.d, #4
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_or8xi64:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: lsr z0.d, z0.d, #4
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <8 x i64>, ptr %a
+ %vb = load <8 x i64>, ptr %b
+ %shift = lshr <8 x i64> %vb, splat(i64 4)
+ %res = or disjoint <8 x i64> %va, %shift
+ store <8 x i64> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_or16xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.d, z0.d, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <16 x i64>, ptr %a
+ %vb = load <16 x i64>, ptr %b
+ %shift = lshr <16 x i64> %vb, splat(i64 4)
+ %res = or disjoint <16 x i64> %va, %shift
+ store <16 x i64> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_or32xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.d, z0.d, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i64>, ptr %a
+ %vb = load <32 x i64>, ptr %b
+ %shift = lshr <32 x i64> %vb, splat(i64 4)
+ %res = or disjoint <32 x i64> %va, %shift
+ store <32 x i64> %res, ptr %a
+ ret void
+}
+
+;
+; USRA legacy memory-form disjoint OR
+;
+
+define void @usra2_disjoint_or32i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra2_disjoint_or32i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.b, z0.b, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i8>, ptr %a, align 32
+ %vb = load <32 x i8>, ptr %b, align 32
+ %shift = lshr <32 x i8> %vb, splat(i8 4)
+ %res = or disjoint <32 x i8> %va, %shift
+ store <32 x i8> %res, ptr %a, align 32
+ ret void
+}
+
+define void @usra2_disjoint_or64i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra2_disjoint_or64i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: mov w8, #32 // =0x20
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
+; CHECK-NEXT: ld1b { z3.b }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.b, z0.b, #4
+; CHECK-NEXT: lsr z1.b, z1.b, #4
+; CHECK-NEXT: orr z0.d, z2.d, z0.d
+; CHECK-NEXT: orr z1.d, z3.d, z1.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0, x8]
+; CHECK-NEXT: st1b { z1.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i8>, ptr %a, align 32
+ %vb = load <64 x i8>, ptr %b, align 32
+ %shift = lshr <64 x i8> %vb, splat(i8 4)
+ %res = or disjoint <64 x i8> %va, %shift
+ store <64 x i8> %res, ptr %a, align 32
+ ret void
+}
+
+define void @usra2_disjoint_or128i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra2_disjoint_or128i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: mov w8, #64 // =0x40
+; CHECK-NEXT: mov w9, #96 // =0x60
+; CHECK-NEXT: mov w10, #32 // =0x20
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1, x9]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x10]
+; CHECK-NEXT: ld1b { z4.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z3.b }, p0/z, [x0, x8]
+; CHECK-NEXT: ld1b { z5.b }, p0/z, [x0, x9]
+; CHECK-NEXT: ld1b { z6.b }, p0/z, [x0, x10]
+; CHECK-NEXT: ld1b { z7.b }, p0/z, [x0]
+; CHECK-NEXT: lsr z0.b, z0.b, #4
+; CHECK-NEXT: lsr z1.b, z1.b, #4
+; CHECK-NEXT: lsr z2.b, z2.b, #4
+; CHECK-NEXT: lsr z4.b, z4.b, #4
+; CHECK-NEXT: orr z0.d, z3.d, z0.d
+; CHECK-NEXT: orr z1.d, z5.d, z1.d
+; CHECK-NEXT: orr z2.d, z6.d, z2.d
+; CHECK-NEXT: orr z3.d, z7.d, z4.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0, x8]
+; CHECK-NEXT: st1b { z1.b }, p0, [x0, x9]
+; CHECK-NEXT: st1b { z2.b }, p0, [x0, x10]
+; CHECK-NEXT: st1b { z3.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i8>, ptr %a, align 32
+ %vb = load <128 x i8>, ptr %b, align 32
+ %shift = lshr <128 x i8> %vb, splat(i8 4)
+ %res = or disjoint <128 x i8> %va, %shift
+ store <128 x i8> %res, ptr %a, align 32
+ ret void
+}
+
+;
+; SSRA disjoint OR
+;
+
+define <8 x i8> @ssra_disjoint_or8xi8(<8 x i8> %a, <8 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or8xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ssra v0.8b, v1.8b, #4
+; CHECK-NEXT: ret
+ %shift = ashr <8 x i8> %b, splat(i8 4)
+ %res = or disjoint <8 x i8> %a, %shift
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @ssra_disjoint_or16xi8(<16 x i8> %a, <16 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or16xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ssra v0.16b, v1.16b, #4
+; CHECK-NEXT: ret
+ %shift = ashr <16 x i8> %b, splat(i8 4)
+ %res = or disjoint <16 x i8> %a, %shift
+ ret <16 x i8> %res
+}
+
+define void @ssra_disjoint_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or32xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: asr z0.b, z0.b, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i8>, ptr %a
+ %vb = load <32 x i8>, ptr %b
+ %shift = ashr <32 x i8> %vb, splat(i8 4)
+ %res = or disjoint <32 x i8> %va, %shift
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or64xi8(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_or64xi8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: asr z0.b, z0.b, #4
+; VBITS_GE_256-NEXT: asr z1.b, z1.b, #4
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_or64xi8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.b, vl64
+; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT: asr z0.b, z0.b, #4
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <64 x i8>, ptr %a
+ %vb = load <64 x i8>, ptr %b
+ %shift = ashr <64 x i8> %vb, splat(i8 4)
+ %res = or disjoint <64 x i8> %va, %shift
+ store <64 x i8> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or128xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl128
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: asr z0.b, z0.b, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i8>, ptr %a
+ %vb = load <128 x i8>, ptr %b
+ %shift = ashr <128 x i8> %vb, splat(i8 4)
+ %res = or disjoint <128 x i8> %va, %shift
+ store <128 x i8> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or256xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl256
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: asr z0.b, z0.b, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <256 x i8>, ptr %a
+ %vb = load <256 x i8>, ptr %b
+ %shift = ashr <256 x i8> %vb, splat(i8 4)
+ %res = or disjoint <256 x i8> %va, %shift
+ store <256 x i8> %res, ptr %a
+ ret void
+}
+
+define <4 x i16> @ssra_disjoint_or4xi16(<4 x i16> %a, <4 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or4xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ssra v0.4h, v1.4h, #4
+; CHECK-NEXT: ret
+ %shift = ashr <4 x i16> %b, splat(i16 4)
+ %res = or disjoint <4 x i16> %a, %shift
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @ssra_disjoint_or8xi16(<8 x i16> %a, <8 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ssra v0.8h, v1.8h, #4
+; CHECK-NEXT: ret
+ %shift = ashr <8 x i16> %b, splat(i16 4)
+ %res = or disjoint <8 x i16> %a, %shift
+ ret <8 x i16> %res
+}
+
+define void @ssra_disjoint_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or16xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl16
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: asr z0.h, z0.h, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <16 x i16>, ptr %a
+ %vb = load <16 x i16>, ptr %b
+ %shift = ashr <16 x i16> %vb, splat(i16 4)
+ %res = or disjoint <16 x i16> %va, %shift
+ store <16 x i16> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or32xi16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_or32xi16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: asr z0.h, z0.h, #4
+; VBITS_GE_256-NEXT: asr z1.h, z1.h, #4
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_or32xi16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.h, vl32
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: asr z0.h, z0.h, #4
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <32 x i16>, ptr %a
+ %vb = load <32 x i16>, ptr %b
+ %shift = ashr <32 x i16> %vb, splat(i16 4)
+ %res = or disjoint <32 x i16> %va, %shift
+ store <32 x i16> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or64xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: asr z0.h, z0.h, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i16>, ptr %a
+ %vb = load <64 x i16>, ptr %b
+ %shift = ashr <64 x i16> %vb, splat(i16 4)
+ %res = or disjoint <64 x i16> %va, %shift
+ store <64 x i16> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or128xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: asr z0.h, z0.h, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i16>, ptr %a
+ %vb = load <128 x i16>, ptr %b
+ %shift = ashr <128 x i16> %vb, splat(i16 4)
+ %res = or disjoint <128 x i16> %va, %shift
+ store <128 x i16> %res, ptr %a
+ ret void
+}
+
+define <2 x i32> @ssra_disjoint_or2xi32(<2 x i32> %a, <2 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or2xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ssra v0.2s, v1.2s, #4
+; CHECK-NEXT: ret
+ %shift = ashr <2 x i32> %b, splat(i32 4)
+ %res = or disjoint <2 x i32> %a, %shift
+ ret <2 x i32> %res
+}
+
+define <4 x i32> @ssra_disjoint_or4xi32(<4 x i32> %a, <4 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ssra v0.4s, v1.4s, #4
+; CHECK-NEXT: ret
+ %shift = ashr <4 x i32> %b, splat(i32 4)
+ %res = or disjoint <4 x i32> %a, %shift
+ ret <4 x i32> %res
+}
+
+define void @ssra_disjoint_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or8xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: asr z0.s, z0.s, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <8 x i32>, ptr %a
+ %vb = load <8 x i32>, ptr %b
+ %shift = ashr <8 x i32> %vb, splat(i32 4)
+ %res = or disjoint <8 x i32> %va, %shift
+ store <8 x i32> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or16xi32(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_or16xi32:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: asr z0.s, z0.s, #4
+; VBITS_GE_256-NEXT: asr z1.s, z1.s, #4
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_or16xi32:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: asr z0.s, z0.s, #4
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <16 x i32>, ptr %a
+ %vb = load <16 x i32>, ptr %b
+ %shift = ashr <16 x i32> %vb, splat(i32 4)
+ %res = or disjoint <16 x i32> %va, %shift
+ store <16 x i32> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or32xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: asr z0.s, z0.s, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i32>, ptr %a
+ %vb = load <32 x i32>, ptr %b
+ %shift = ashr <32 x i32> %vb, splat(i32 4)
+ %res = or disjoint <32 x i32> %va, %shift
+ store <32 x i32> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or64xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: asr z0.s, z0.s, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i32>, ptr %a
+ %vb = load <64 x i32>, ptr %b
+ %shift = ashr <64 x i32> %vb, splat(i32 4)
+ %res = or disjoint <64 x i32> %va, %shift
+ store <64 x i32> %res, ptr %a
+ ret void
+}
+
+define <1 x i64> @ssra_disjoint_or1xi64(<1 x i64> %a, <1 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or1xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ssra d0, d1, #4
+; CHECK-NEXT: ret
+ %shift = ashr <1 x i64> %b, splat(i64 4)
+ %res = or disjoint <1 x i64> %a, %shift
+ ret <1 x i64> %res
+}
+
+define <2 x i64> @ssra_disjoint_or2xi64(<2 x i64> %a, <2 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or2xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ssra v0.2d, v1.2d, #4
+; CHECK-NEXT: ret
+ %shift = ashr <2 x i64> %b, splat(i64 4)
+ %res = or disjoint <2 x i64> %a, %shift
+ ret <2 x i64> %res
+}
+
+define void @ssra_disjoint_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or4xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: asr z0.d, z0.d, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <4 x i64>, ptr %a
+ %vb = load <4 x i64>, ptr %b
+ %shift = ashr <4 x i64> %vb, splat(i64 4)
+ %res = or disjoint <4 x i64> %va, %shift
+ store <4 x i64> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or8xi64(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_or8xi64:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: asr z0.d, z0.d, #4
+; VBITS_GE_256-NEXT: asr z1.d, z1.d, #4
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_or8xi64:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: asr z0.d, z0.d, #4
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <8 x i64>, ptr %a
+ %vb = load <8 x i64>, ptr %b
+ %shift = ashr <8 x i64> %vb, splat(i64 4)
+ %res = or disjoint <8 x i64> %va, %shift
+ store <8 x i64> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or16xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: asr z0.d, z0.d, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <16 x i64>, ptr %a
+ %vb = load <16 x i64>, ptr %b
+ %shift = ashr <16 x i64> %vb, splat(i64 4)
+ %res = or disjoint <16 x i64> %va, %shift
+ store <16 x i64> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or32xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: asr z0.d, z0.d, #4
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i64>, ptr %a
+ %vb = load <32 x i64>, ptr %b
+ %shift = ashr <32 x i64> %vb, splat(i64 4)
+ %res = or disjoint <32 x i64> %va, %shift
+ store <32 x i64> %res, ptr %a
+ ret void
+}
+
+;
+; USRA disjoint shifted OR
+;
+
+define <8 x i8> @usra_disjoint_shift_or8xi8(<8 x i8> %a, <8 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or8xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: usra v0.8b, v1.8b, #1
+; CHECK-NEXT: ret
+ %shl = shl <8 x i8> %a, splat(i8 7)
+ %srl = lshr <8 x i8> %b, splat(i8 1)
+ %res = or <8 x i8> %shl, %srl
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @usra_disjoint_shift_or16xi8(<16 x i8> %a, <16 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or16xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: usra v0.16b, v1.16b, #1
+; CHECK-NEXT: ret
+ %shl = shl <16 x i8> %a, splat(i8 7)
+ %srl = lshr <16 x i8> %b, splat(i8 1)
+ %res = or <16 x i8> %shl, %srl
+ ret <16 x i8> %res
+}
+
+define void @usra_disjoint_shift_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or32xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.b, z0.b, #7
+; CHECK-NEXT: lsr z1.b, z1.b, #1
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i8>, ptr %a
+ %vb = load <32 x i8>, ptr %b
+ %shl = shl <32 x i8> %va, splat(i8 7)
+ %srl = lshr <32 x i8> %vb, splat(i8 1)
+ %res = or <32 x i8> %shl, %srl
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or64xi8(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_shift_or64xi8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT: lsl z0.b, z0.b, #7
+; VBITS_GE_256-NEXT: lsr z1.b, z1.b, #1
+; VBITS_GE_256-NEXT: lsl z2.b, z2.b, #7
+; VBITS_GE_256-NEXT: lsr z3.b, z3.b, #1
+; VBITS_GE_256-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_256-NEXT: orr z1.d, z2.d, z3.d
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_shift_or64xi8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.b, vl64
+; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: lsl z0.b, z0.b, #7
+; VBITS_GE_512-NEXT: lsr z1.b, z1.b, #1
+; VBITS_GE_512-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <64 x i8>, ptr %a
+ %vb = load <64 x i8>, ptr %b
+ %shl = shl <64 x i8> %va, splat(i8 7)
+ %srl = lshr <64 x i8> %vb, splat(i8 1)
+ %res = or <64 x i8> %shl, %srl
+ store <64 x i8> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or128xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl128
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.b, z0.b, #7
+; CHECK-NEXT: lsr z1.b, z1.b, #1
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i8>, ptr %a
+ %vb = load <128 x i8>, ptr %b
+ %shl = shl <128 x i8> %va, splat(i8 7)
+ %srl = lshr <128 x i8> %vb, splat(i8 1)
+ %res = or <128 x i8> %shl, %srl
+ store <128 x i8> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or256xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl256
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.b, z0.b, #7
+; CHECK-NEXT: lsr z1.b, z1.b, #1
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <256 x i8>, ptr %a
+ %vb = load <256 x i8>, ptr %b
+ %shl = shl <256 x i8> %va, splat(i8 7)
+ %srl = lshr <256 x i8> %vb, splat(i8 1)
+ %res = or <256 x i8> %shl, %srl
+ store <256 x i8> %res, ptr %a
+ ret void
+}
+
+define <4 x i16> @usra_disjoint_shift_or4xi16(<4 x i16> %a, <4 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or4xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #7
+; CHECK-NEXT: usra v0.4h, v1.4h, #9
+; CHECK-NEXT: ret
+ %shl = shl <4 x i16> %a, splat(i16 7)
+ %srl = lshr <4 x i16> %b, splat(i16 9)
+ %res = or <4 x i16> %shl, %srl
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @usra_disjoint_shift_or8xi16(<8 x i16> %a, <8 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8h, v0.8h, #7
+; CHECK-NEXT: usra v0.8h, v1.8h, #9
+; CHECK-NEXT: ret
+ %shl = shl <8 x i16> %a, splat(i16 7)
+ %srl = lshr <8 x i16> %b, splat(i16 9)
+ %res = or <8 x i16> %shl, %srl
+ ret <8 x i16> %res
+}
+
+define void @usra_disjoint_shift_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or16xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl16
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.h, z0.h, #7
+; CHECK-NEXT: lsr z1.h, z1.h, #9
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <16 x i16>, ptr %a
+ %vb = load <16 x i16>, ptr %b
+ %shl = shl <16 x i16> %va, splat(i16 7)
+ %srl = lshr <16 x i16> %vb, splat(i16 9)
+ %res = or <16 x i16> %shl, %srl
+ store <16 x i16> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or32xi16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_shift_or32xi16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: lsl z0.h, z0.h, #7
+; VBITS_GE_256-NEXT: lsr z1.h, z1.h, #9
+; VBITS_GE_256-NEXT: lsl z2.h, z2.h, #7
+; VBITS_GE_256-NEXT: lsr z3.h, z3.h, #9
+; VBITS_GE_256-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_256-NEXT: orr z1.d, z2.d, z3.d
+; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_shift_or32xi16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.h, vl32
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: lsl z0.h, z0.h, #7
+; VBITS_GE_512-NEXT: lsr z1.h, z1.h, #9
+; VBITS_GE_512-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <32 x i16>, ptr %a
+ %vb = load <32 x i16>, ptr %b
+ %shl = shl <32 x i16> %va, splat(i16 7)
+ %srl = lshr <32 x i16> %vb, splat(i16 9)
+ %res = or <32 x i16> %shl, %srl
+ store <32 x i16> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or64xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.h, z0.h, #7
+; CHECK-NEXT: lsr z1.h, z1.h, #9
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i16>, ptr %a
+ %vb = load <64 x i16>, ptr %b
+ %shl = shl <64 x i16> %va, splat(i16 7)
+ %srl = lshr <64 x i16> %vb, splat(i16 9)
+ %res = or <64 x i16> %shl, %srl
+ store <64 x i16> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or128xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.h, z0.h, #7
+; CHECK-NEXT: lsr z1.h, z1.h, #9
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i16>, ptr %a
+ %vb = load <128 x i16>, ptr %b
+ %shl = shl <128 x i16> %va, splat(i16 7)
+ %srl = lshr <128 x i16> %vb, splat(i16 9)
+ %res = or <128 x i16> %shl, %srl
+ store <128 x i16> %res, ptr %a
+ ret void
+}
+
+define <2 x i32> @usra_disjoint_shift_or2xi32(<2 x i32> %a, <2 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or2xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #7
+; CHECK-NEXT: usra v0.2s, v1.2s, #25
+; CHECK-NEXT: ret
+ %shl = shl <2 x i32> %a, splat(i32 7)
+ %srl = lshr <2 x i32> %b, splat(i32 25)
+ %res = or <2 x i32> %shl, %srl
+ ret <2 x i32> %res
+}
+
+define <4 x i32> @usra_disjoint_shift_or4xi32(<4 x i32> %a, <4 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4s, v0.4s, #7
+; CHECK-NEXT: usra v0.4s, v1.4s, #25
+; CHECK-NEXT: ret
+ %shl = shl <4 x i32> %a, splat(i32 7)
+ %srl = lshr <4 x i32> %b, splat(i32 25)
+ %res = or <4 x i32> %shl, %srl
+ ret <4 x i32> %res
+}
+
+define void @usra_disjoint_shift_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or8xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.s, z0.s, #7
+; CHECK-NEXT: lsr z1.s, z1.s, #25
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <8 x i32>, ptr %a
+ %vb = load <8 x i32>, ptr %b
+ %shl = shl <8 x i32> %va, splat(i32 7)
+ %srl = lshr <8 x i32> %vb, splat(i32 25)
+ %res = or <8 x i32> %shl, %srl
+ store <8 x i32> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or16xi32(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_shift_or16xi32:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: lsl z0.s, z0.s, #7
+; VBITS_GE_256-NEXT: lsr z1.s, z1.s, #25
+; VBITS_GE_256-NEXT: lsl z2.s, z2.s, #7
+; VBITS_GE_256-NEXT: lsr z3.s, z3.s, #25
+; VBITS_GE_256-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_256-NEXT: orr z1.d, z2.d, z3.d
+; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_shift_or16xi32:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: lsl z0.s, z0.s, #7
+; VBITS_GE_512-NEXT: lsr z1.s, z1.s, #25
+; VBITS_GE_512-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <16 x i32>, ptr %a
+ %vb = load <16 x i32>, ptr %b
+ %shl = shl <16 x i32> %va, splat(i32 7)
+ %srl = lshr <16 x i32> %vb, splat(i32 25)
+ %res = or <16 x i32> %shl, %srl
+ store <16 x i32> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or32xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.s, z0.s, #7
+; CHECK-NEXT: lsr z1.s, z1.s, #25
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i32>, ptr %a
+ %vb = load <32 x i32>, ptr %b
+ %shl = shl <32 x i32> %va, splat(i32 7)
+ %srl = lshr <32 x i32> %vb, splat(i32 25)
+ %res = or <32 x i32> %shl, %srl
+ store <32 x i32> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or64xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.s, z0.s, #7
+; CHECK-NEXT: lsr z1.s, z1.s, #25
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i32>, ptr %a
+ %vb = load <64 x i32>, ptr %b
+ %shl = shl <64 x i32> %va, splat(i32 7)
+ %srl = lshr <64 x i32> %vb, splat(i32 25)
+ %res = or <64 x i32> %shl, %srl
+ store <64 x i32> %res, ptr %a
+ ret void
+}
+
+define <1 x i64> @usra_disjoint_shift_or1xi64(<1 x i64> %a, <1 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or1xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl d0, d0, #7
+; CHECK-NEXT: usra d0, d1, #57
+; CHECK-NEXT: ret
+ %shl = shl <1 x i64> %a, splat(i64 7)
+ %srl = lshr <1 x i64> %b, splat(i64 57)
+ %res = or <1 x i64> %shl, %srl
+ ret <1 x i64> %res
+}
+
+define <2 x i64> @usra_disjoint_shift_or2xi64(<2 x i64> %a, <2 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or2xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2d, v0.2d, #7
+; CHECK-NEXT: usra v0.2d, v1.2d, #57
+; CHECK-NEXT: ret
+ %shl = shl <2 x i64> %a, splat(i64 7)
+ %srl = lshr <2 x i64> %b, splat(i64 57)
+ %res = or <2 x i64> %shl, %srl
+ ret <2 x i64> %res
+}
+
+define void @usra_disjoint_shift_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or4xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.d, z0.d, #7
+; CHECK-NEXT: lsr z1.d, z1.d, #57
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <4 x i64>, ptr %a
+ %vb = load <4 x i64>, ptr %b
+ %shl = shl <4 x i64> %va, splat(i64 7)
+ %srl = lshr <4 x i64> %vb, splat(i64 57)
+ %res = or <4 x i64> %shl, %srl
+ store <4 x i64> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or8xi64(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_shift_or8xi64:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: lsl z0.d, z0.d, #7
+; VBITS_GE_256-NEXT: lsr z1.d, z1.d, #57
+; VBITS_GE_256-NEXT: lsl z2.d, z2.d, #7
+; VBITS_GE_256-NEXT: lsr z3.d, z3.d, #57
+; VBITS_GE_256-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_256-NEXT: orr z1.d, z2.d, z3.d
+; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_shift_or8xi64:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: lsl z0.d, z0.d, #7
+; VBITS_GE_512-NEXT: lsr z1.d, z1.d, #57
+; VBITS_GE_512-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <8 x i64>, ptr %a
+ %vb = load <8 x i64>, ptr %b
+ %shl = shl <8 x i64> %va, splat(i64 7)
+ %srl = lshr <8 x i64> %vb, splat(i64 57)
+ %res = or <8 x i64> %shl, %srl
+ store <8 x i64> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or16xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.d, z0.d, #7
+; CHECK-NEXT: lsr z1.d, z1.d, #57
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <16 x i64>, ptr %a
+ %vb = load <16 x i64>, ptr %b
+ %shl = shl <16 x i64> %va, splat(i64 7)
+ %srl = lshr <16 x i64> %vb, splat(i64 57)
+ %res = or <16 x i64> %shl, %srl
+ store <16 x i64> %res, ptr %a
+ ret void
+}
+
+define void @usra_disjoint_shift_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or32xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: lsl z0.d, z0.d, #7
+; CHECK-NEXT: lsr z1.d, z1.d, #57
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i64>, ptr %a
+ %vb = load <32 x i64>, ptr %b
+ %shl = shl <32 x i64> %va, splat(i64 7)
+ %srl = lshr <32 x i64> %vb, splat(i64 57)
+ %res = or <32 x i64> %shl, %srl
+ store <32 x i64> %res, ptr %a
+ ret void
+}
+
+;
+; SSRA disjoint shifted OR
+;
+
+define <8 x i8> @ssra_disjoint_shift_or8xi8(<8 x i8> %a, <8 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or8xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.8b, v0.8b, #2
+; CHECK-NEXT: shl v1.8b, v1.8b, #7
+; CHECK-NEXT: ssra v0.8b, v1.8b, #1
+; CHECK-NEXT: ret
+ %acc = lshr <8 x i8> %a, splat(i8 2)
+ %sign = shl <8 x i8> %b, splat(i8 7)
+ %sra = ashr <8 x i8> %sign, splat(i8 1)
+ %res = or <8 x i8> %acc, %sra
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @ssra_disjoint_shift_or16xi8(<16 x i8> %a, <16 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or16xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.16b, v0.16b, #2
+; CHECK-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-NEXT: ssra v0.16b, v1.16b, #1
+; CHECK-NEXT: ret
+ %acc = lshr <16 x i8> %a, splat(i8 2)
+ %sign = shl <16 x i8> %b, splat(i8 7)
+ %sra = ashr <16 x i8> %sign, splat(i8 1)
+ %res = or <16 x i8> %acc, %sra
+ ret <16 x i8> %res
+}
+
+define void @ssra_disjoint_shift_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or32xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.b, z0.b, #7
+; CHECK-NEXT: lsr z1.b, z1.b, #2
+; CHECK-NEXT: asr z0.b, z0.b, #1
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i8>, ptr %a
+ %vb = load <32 x i8>, ptr %b
+ %acc = lshr <32 x i8> %va, splat(i8 2)
+ %sign = shl <32 x i8> %vb, splat(i8 7)
+ %sra = ashr <32 x i8> %sign, splat(i8 1)
+ %res = or <32 x i8> %acc, %sra
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or64xi8(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_shift_or64xi8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: lsl z0.b, z0.b, #7
+; VBITS_GE_256-NEXT: lsl z1.b, z1.b, #7
+; VBITS_GE_256-NEXT: lsr z2.b, z2.b, #2
+; VBITS_GE_256-NEXT: lsr z3.b, z3.b, #2
+; VBITS_GE_256-NEXT: asr z0.b, z0.b, #1
+; VBITS_GE_256-NEXT: asr z1.b, z1.b, #1
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_shift_or64xi8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.b, vl64
+; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT: lsl z0.b, z0.b, #7
+; VBITS_GE_512-NEXT: lsr z1.b, z1.b, #2
+; VBITS_GE_512-NEXT: asr z0.b, z0.b, #1
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <64 x i8>, ptr %a
+ %vb = load <64 x i8>, ptr %b
+ %acc = lshr <64 x i8> %va, splat(i8 2)
+ %sign = shl <64 x i8> %vb, splat(i8 7)
+ %sra = ashr <64 x i8> %sign, splat(i8 1)
+ %res = or <64 x i8> %acc, %sra
+ store <64 x i8> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or128xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl128
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.b, z0.b, #7
+; CHECK-NEXT: lsr z1.b, z1.b, #2
+; CHECK-NEXT: asr z0.b, z0.b, #1
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i8>, ptr %a
+ %vb = load <128 x i8>, ptr %b
+ %acc = lshr <128 x i8> %va, splat(i8 2)
+ %sign = shl <128 x i8> %vb, splat(i8 7)
+ %sra = ashr <128 x i8> %sign, splat(i8 1)
+ %res = or <128 x i8> %acc, %sra
+ store <128 x i8> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or256xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl256
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.b, z0.b, #7
+; CHECK-NEXT: lsr z1.b, z1.b, #2
+; CHECK-NEXT: asr z0.b, z0.b, #1
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <256 x i8>, ptr %a
+ %vb = load <256 x i8>, ptr %b
+ %acc = lshr <256 x i8> %va, splat(i8 2)
+ %sign = shl <256 x i8> %vb, splat(i8 7)
+ %sra = ashr <256 x i8> %sign, splat(i8 1)
+ %res = or <256 x i8> %acc, %sra
+ store <256 x i8> %res, ptr %a
+ ret void
+}
+
+define <4 x i16> @ssra_disjoint_shift_or4xi16(<4 x i16> %a, <4 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or4xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.4h, v0.4h, #10
+; CHECK-NEXT: shl v1.4h, v1.4h, #15
+; CHECK-NEXT: ssra v0.4h, v1.4h, #9
+; CHECK-NEXT: ret
+ %acc = lshr <4 x i16> %a, splat(i16 10)
+ %sign = shl <4 x i16> %b, splat(i16 15)
+ %sra = ashr <4 x i16> %sign, splat(i16 9)
+ %res = or <4 x i16> %acc, %sra
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @ssra_disjoint_shift_or8xi16(<8 x i16> %a, <8 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.8h, v0.8h, #10
+; CHECK-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-NEXT: ssra v0.8h, v1.8h, #9
+; CHECK-NEXT: ret
+ %acc = lshr <8 x i16> %a, splat(i16 10)
+ %sign = shl <8 x i16> %b, splat(i16 15)
+ %sra = ashr <8 x i16> %sign, splat(i16 9)
+ %res = or <8 x i16> %acc, %sra
+ ret <8 x i16> %res
+}
+
+define void @ssra_disjoint_shift_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or16xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl16
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.h, z0.h, #15
+; CHECK-NEXT: lsr z1.h, z1.h, #10
+; CHECK-NEXT: asr z0.h, z0.h, #9
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <16 x i16>, ptr %a
+ %vb = load <16 x i16>, ptr %b
+ %acc = lshr <16 x i16> %va, splat(i16 10)
+ %sign = shl <16 x i16> %vb, splat(i16 15)
+ %sra = ashr <16 x i16> %sign, splat(i16 9)
+ %res = or <16 x i16> %acc, %sra
+ store <16 x i16> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or32xi16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_shift_or32xi16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: lsl z0.h, z0.h, #15
+; VBITS_GE_256-NEXT: lsl z1.h, z1.h, #15
+; VBITS_GE_256-NEXT: lsr z2.h, z2.h, #10
+; VBITS_GE_256-NEXT: lsr z3.h, z3.h, #10
+; VBITS_GE_256-NEXT: asr z0.h, z0.h, #9
+; VBITS_GE_256-NEXT: asr z1.h, z1.h, #9
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_shift_or32xi16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.h, vl32
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: lsl z0.h, z0.h, #15
+; VBITS_GE_512-NEXT: lsr z1.h, z1.h, #10
+; VBITS_GE_512-NEXT: asr z0.h, z0.h, #9
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <32 x i16>, ptr %a
+ %vb = load <32 x i16>, ptr %b
+ %acc = lshr <32 x i16> %va, splat(i16 10)
+ %sign = shl <32 x i16> %vb, splat(i16 15)
+ %sra = ashr <32 x i16> %sign, splat(i16 9)
+ %res = or <32 x i16> %acc, %sra
+ store <32 x i16> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or64xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.h, z0.h, #15
+; CHECK-NEXT: lsr z1.h, z1.h, #10
+; CHECK-NEXT: asr z0.h, z0.h, #9
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i16>, ptr %a
+ %vb = load <64 x i16>, ptr %b
+ %acc = lshr <64 x i16> %va, splat(i16 10)
+ %sign = shl <64 x i16> %vb, splat(i16 15)
+ %sra = ashr <64 x i16> %sign, splat(i16 9)
+ %res = or <64 x i16> %acc, %sra
+ store <64 x i16> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or128xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.h, z0.h, #15
+; CHECK-NEXT: lsr z1.h, z1.h, #10
+; CHECK-NEXT: asr z0.h, z0.h, #9
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <128 x i16>, ptr %a
+ %vb = load <128 x i16>, ptr %b
+ %acc = lshr <128 x i16> %va, splat(i16 10)
+ %sign = shl <128 x i16> %vb, splat(i16 15)
+ %sra = ashr <128 x i16> %sign, splat(i16 9)
+ %res = or <128 x i16> %acc, %sra
+ store <128 x i16> %res, ptr %a
+ ret void
+}
+
+define <2 x i32> @ssra_disjoint_shift_or2xi32(<2 x i32> %a, <2 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or2xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.2s, v0.2s, #26
+; CHECK-NEXT: shl v1.2s, v1.2s, #31
+; CHECK-NEXT: ssra v0.2s, v1.2s, #25
+; CHECK-NEXT: ret
+ %acc = lshr <2 x i32> %a, splat(i32 26)
+ %sign = shl <2 x i32> %b, splat(i32 31)
+ %sra = ashr <2 x i32> %sign, splat(i32 25)
+ %res = or <2 x i32> %acc, %sra
+ ret <2 x i32> %res
+}
+
+define <4 x i32> @ssra_disjoint_shift_or4xi32(<4 x i32> %a, <4 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.4s, v0.4s, #26
+; CHECK-NEXT: shl v1.4s, v1.4s, #31
+; CHECK-NEXT: ssra v0.4s, v1.4s, #25
+; CHECK-NEXT: ret
+ %acc = lshr <4 x i32> %a, splat(i32 26)
+ %sign = shl <4 x i32> %b, splat(i32 31)
+ %sra = ashr <4 x i32> %sign, splat(i32 25)
+ %res = or <4 x i32> %acc, %sra
+ ret <4 x i32> %res
+}
+
+define void @ssra_disjoint_shift_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or8xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.s, z0.s, #31
+; CHECK-NEXT: lsr z1.s, z1.s, #26
+; CHECK-NEXT: asr z0.s, z0.s, #25
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <8 x i32>, ptr %a
+ %vb = load <8 x i32>, ptr %b
+ %acc = lshr <8 x i32> %va, splat(i32 26)
+ %sign = shl <8 x i32> %vb, splat(i32 31)
+ %sra = ashr <8 x i32> %sign, splat(i32 25)
+ %res = or <8 x i32> %acc, %sra
+ store <8 x i32> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or16xi32(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_shift_or16xi32:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: lsl z0.s, z0.s, #31
+; VBITS_GE_256-NEXT: lsl z1.s, z1.s, #31
+; VBITS_GE_256-NEXT: lsr z2.s, z2.s, #26
+; VBITS_GE_256-NEXT: lsr z3.s, z3.s, #26
+; VBITS_GE_256-NEXT: asr z0.s, z0.s, #25
+; VBITS_GE_256-NEXT: asr z1.s, z1.s, #25
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_shift_or16xi32:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: lsl z0.s, z0.s, #31
+; VBITS_GE_512-NEXT: lsr z1.s, z1.s, #26
+; VBITS_GE_512-NEXT: asr z0.s, z0.s, #25
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <16 x i32>, ptr %a
+ %vb = load <16 x i32>, ptr %b
+ %acc = lshr <16 x i32> %va, splat(i32 26)
+ %sign = shl <16 x i32> %vb, splat(i32 31)
+ %sra = ashr <16 x i32> %sign, splat(i32 25)
+ %res = or <16 x i32> %acc, %sra
+ store <16 x i32> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or32xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.s, z0.s, #31
+; CHECK-NEXT: lsr z1.s, z1.s, #26
+; CHECK-NEXT: asr z0.s, z0.s, #25
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i32>, ptr %a
+ %vb = load <32 x i32>, ptr %b
+ %acc = lshr <32 x i32> %va, splat(i32 26)
+ %sign = shl <32 x i32> %vb, splat(i32 31)
+ %sra = ashr <32 x i32> %sign, splat(i32 25)
+ %res = or <32 x i32> %acc, %sra
+ store <32 x i32> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or64xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.s, z0.s, #31
+; CHECK-NEXT: lsr z1.s, z1.s, #26
+; CHECK-NEXT: asr z0.s, z0.s, #25
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <64 x i32>, ptr %a
+ %vb = load <64 x i32>, ptr %b
+ %acc = lshr <64 x i32> %va, splat(i32 26)
+ %sign = shl <64 x i32> %vb, splat(i32 31)
+ %sra = ashr <64 x i32> %sign, splat(i32 25)
+ %res = or <64 x i32> %acc, %sra
+ store <64 x i32> %res, ptr %a
+ ret void
+}
+
+define <1 x i64> @ssra_disjoint_shift_or1xi64(<1 x i64> %a, <1 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or1xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr d0, d0, #58
+; CHECK-NEXT: shl d1, d1, #63
+; CHECK-NEXT: ssra d0, d1, #57
+; CHECK-NEXT: ret
+ %acc = lshr <1 x i64> %a, splat(i64 58)
+ %sign = shl <1 x i64> %b, splat(i64 63)
+ %sra = ashr <1 x i64> %sign, splat(i64 57)
+ %res = or <1 x i64> %acc, %sra
+ ret <1 x i64> %res
+}
+
+define <2 x i64> @ssra_disjoint_shift_or2xi64(<2 x i64> %a, <2 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or2xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.2d, v0.2d, #58
+; CHECK-NEXT: shl v1.2d, v1.2d, #63
+; CHECK-NEXT: ssra v0.2d, v1.2d, #57
+; CHECK-NEXT: ret
+ %acc = lshr <2 x i64> %a, splat(i64 58)
+ %sign = shl <2 x i64> %b, splat(i64 63)
+ %sra = ashr <2 x i64> %sign, splat(i64 57)
+ %res = or <2 x i64> %acc, %sra
+ ret <2 x i64> %res
+}
+
+define void @ssra_disjoint_shift_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or4xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.d, z0.d, #63
+; CHECK-NEXT: lsr z1.d, z1.d, #58
+; CHECK-NEXT: asr z0.d, z0.d, #57
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <4 x i64>, ptr %a
+ %vb = load <4 x i64>, ptr %b
+ %acc = lshr <4 x i64> %va, splat(i64 58)
+ %sign = shl <4 x i64> %vb, splat(i64 63)
+ %sra = ashr <4 x i64> %sign, splat(i64 57)
+ %res = or <4 x i64> %acc, %sra
+ store <4 x i64> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or8xi64(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_shift_or8xi64:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: lsl z0.d, z0.d, #63
+; VBITS_GE_256-NEXT: lsl z1.d, z1.d, #63
+; VBITS_GE_256-NEXT: lsr z2.d, z2.d, #58
+; VBITS_GE_256-NEXT: lsr z3.d, z3.d, #58
+; VBITS_GE_256-NEXT: asr z0.d, z0.d, #57
+; VBITS_GE_256-NEXT: asr z1.d, z1.d, #57
+; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
+; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_shift_or8xi64:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: lsl z0.d, z0.d, #63
+; VBITS_GE_512-NEXT: lsr z1.d, z1.d, #58
+; VBITS_GE_512-NEXT: asr z0.d, z0.d, #57
+; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %va = load <8 x i64>, ptr %a
+ %vb = load <8 x i64>, ptr %b
+ %acc = lshr <8 x i64> %va, splat(i64 58)
+ %sign = shl <8 x i64> %vb, splat(i64 63)
+ %sra = ashr <8 x i64> %sign, splat(i64 57)
+ %res = or <8 x i64> %acc, %sra
+ store <8 x i64> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or16xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.d, z0.d, #63
+; CHECK-NEXT: lsr z1.d, z1.d, #58
+; CHECK-NEXT: asr z0.d, z0.d, #57
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <16 x i64>, ptr %a
+ %vb = load <16 x i64>, ptr %b
+ %acc = lshr <16 x i64> %va, splat(i64 58)
+ %sign = shl <16 x i64> %vb, splat(i64 63)
+ %sra = ashr <16 x i64> %sign, splat(i64 57)
+ %res = or <16 x i64> %acc, %sra
+ store <16 x i64> %res, ptr %a
+ ret void
+}
+
+define void @ssra_disjoint_shift_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or32xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: lsl z0.d, z0.d, #63
+; CHECK-NEXT: lsr z1.d, z1.d, #58
+; CHECK-NEXT: asr z0.d, z0.d, #57
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %va = load <32 x i64>, ptr %a
+ %vb = load <32 x i64>, ptr %b
+ %acc = lshr <32 x i64> %va, splat(i64 58)
+ %sign = shl <32 x i64> %vb, splat(i64 63)
+ %sra = ashr <32 x i64> %sign, splat(i64 57)
+ %res = or <32 x i64> %acc, %sra
+ store <32 x i64> %res, ptr %a
+ ret void
+}
+
+declare <16 x i8> @llvm.aarch64.neon.ushl.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.ushl.v2i64(<2 x i64>, <2 x i64>)
+
+declare <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.aarch64.neon.sshl.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64>, <2 x i64>)
+
+attributes #0 = { "target-features"="+sve,+sve2" }
diff --git a/llvm/test/CodeGen/AArch64/sve2-sra.ll b/llvm/test/CodeGen/AArch64/sve2-sra.ll
index 0b951b01a5e90..5e5f4234e5e06 100644
--- a/llvm/test/CodeGen/AArch64/sve2-sra.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-sra.ll
@@ -255,6 +255,8 @@ define <vscale x 2 x i64> @ssra_intr_u_i64(<vscale x 2 x i1> %pg, <vscale x 2 x
ret <vscale x 2 x i64> %add
}
+; USRA
+
define <vscale x 16 x i8> @usra_disjoint_or16xi8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
; CHECK-LABEL: usra_disjoint_or16xi8:
; CHECK: // %bb.0:
@@ -335,6 +337,244 @@ define <vscale x 2 x i64> @ssra_disjoint_or2xi64(<vscale x 2 x i64> %a, <vscale
ret <vscale x 2 x i64> %add
}
+define <vscale x 16 x i8> @usra_disjoint_shift_or16xi8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or16xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.b, z0.b, #7
+; CHECK-NEXT: lsr z1.b, z1.b, #1
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %shl = shl <vscale x 16 x i8> %a, splat (i8 7)
+ %srl = lshr <vscale x 16 x i8> %b, splat (i8 1)
+ %r = or <vscale x 16 x i8> %shl, %srl
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 8 x i16> @usra_disjoint_shift_or8xi16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.h, z0.h, #7
+; CHECK-NEXT: lsr z1.h, z1.h, #9
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %shl = shl <vscale x 8 x i16> %a, splat (i16 7)
+ %srl = lshr <vscale x 8 x i16> %b, splat (i16 9)
+ %r = or <vscale x 8 x i16> %shl, %srl
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 4 x i32> @usra_disjoint_shift_or4xi32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #7
+; CHECK-NEXT: lsr z1.s, z1.s, #25
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %shl = shl <vscale x 4 x i32> %a, splat (i32 7)
+ %srl = lshr <vscale x 4 x i32> %b, splat (i32 25)
+ %r = or <vscale x 4 x i32> %shl, %srl
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 2 x i64> @usra_disjoint_shift_or2xi64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+; CHECK-LABEL: usra_disjoint_shift_or2xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.d, z0.d, #7
+; CHECK-NEXT: lsr z1.d, z1.d, #57
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %shl = shl <vscale x 2 x i64> %a, splat (i64 7)
+ %srl = lshr <vscale x 2 x i64> %b, splat (i64 57)
+ %r = or <vscale x 2 x i64> %shl, %srl
+ ret <vscale x 2 x i64> %r
+}
+
+; SSRA
+
+define <vscale x 16 x i8> @ssra_disjoint_shift_or16xi8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or16xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.b, z1.b, #7
+; CHECK-NEXT: lsr z0.b, z0.b, #2
+; CHECK-NEXT: asr z1.b, z1.b, #1
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %acc = lshr <vscale x 16 x i8> %a, splat (i8 2)
+ %sign = shl <vscale x 16 x i8> %b, splat (i8 7)
+ %sra = ashr <vscale x 16 x i8> %sign, splat (i8 1)
+ %r = or <vscale x 16 x i8> %acc, %sra
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 8 x i16> @ssra_disjoint_shift_or8xi16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.h, z1.h, #15
+; CHECK-NEXT: lsr z0.h, z0.h, #10
+; CHECK-NEXT: asr z1.h, z1.h, #9
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %acc = lshr <vscale x 8 x i16> %a, splat (i16 10)
+ %sign = shl <vscale x 8 x i16> %b, splat (i16 15)
+ %sra = ashr <vscale x 8 x i16> %sign, splat (i16 9)
+ %r = or <vscale x 8 x i16> %acc, %sra
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 4 x i32> @ssra_disjoint_shift_or4xi32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: lsr z0.s, z0.s, #26
+; CHECK-NEXT: asr z1.s, z1.s, #25
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %acc = lshr <vscale x 4 x i32> %a, splat (i32 26)
+ %sign = shl <vscale x 4 x i32> %b, splat (i32 31)
+ %sra = ashr <vscale x 4 x i32> %sign, splat (i32 25)
+ %r = or <vscale x 4 x i32> %acc, %sra
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 2 x i64> @ssra_disjoint_shift_or2xi64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_or2xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.d, z1.d, #63
+; CHECK-NEXT: lsr z0.d, z0.d, #58
+; CHECK-NEXT: asr z1.d, z1.d, #57
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %acc = lshr <vscale x 2 x i64> %a, splat (i64 58)
+ %sign = shl <vscale x 2 x i64> %b, splat (i64 63)
+ %sra = ashr <vscale x 2 x i64> %sign, splat (i64 57)
+ %r = or <vscale x 2 x i64> %acc, %sra
+ ret <vscale x 2 x i64> %r
+}
+
+; USRA
+
+define <vscale x 16 x i8> @usra_disjoint_shl_lsr_or16xi8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+; CHECK-LABEL: usra_disjoint_shl_lsr_or16xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.b, z0.b, #4
+; CHECK-NEXT: lsr z1.b, z1.b, #4
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %ptrue = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+ %shl = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.u.nxv16i8(<vscale x 16 x i1> %ptrue, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 4))
+ %srl = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %ptrue, <vscale x 16 x i8> %b, <vscale x 16 x i8> splat(i8 4))
+ %r = or <vscale x 16 x i8> %shl, %srl
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 8 x i16> @usra_disjoint_shl_lsr_or8xi16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+; CHECK-LABEL: usra_disjoint_shl_lsr_or8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.h, z0.h, #8
+; CHECK-NEXT: lsr z1.h, z1.h, #8
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %ptrue = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+ %shl = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.u.nxv8i16(<vscale x 8 x i1> %ptrue, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 8))
+ %srl = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %ptrue, <vscale x 8 x i16> %b, <vscale x 8 x i16> splat(i16 8))
+ %r = or <vscale x 8 x i16> %shl, %srl
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 4 x i32> @usra_disjoint_shl_lsr_or4xi32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+; CHECK-LABEL: usra_disjoint_shl_lsr_or4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsr z1.s, z1.s, #16
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %ptrue = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+ %shl = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.u.nxv4i32(<vscale x 4 x i1> %ptrue, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 16))
+ %srl = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %ptrue, <vscale x 4 x i32> %b, <vscale x 4 x i32> splat(i32 16))
+ %r = or <vscale x 4 x i32> %shl, %srl
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 2 x i64> @usra_disjoint_shl_lsr_or2xi64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+; CHECK-LABEL: usra_disjoint_shl_lsr_or2xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.d, z0.d, #32
+; CHECK-NEXT: lsr z1.d, z1.d, #32
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %ptrue = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+ %shl = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.u.nxv2i64(<vscale x 2 x i1> %ptrue, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 32))
+ %srl = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %ptrue, <vscale x 2 x i64> %b, <vscale x 2 x i64> splat(i64 32))
+ %r = or <vscale x 2 x i64> %shl, %srl
+ ret <vscale x 2 x i64> %r
+}
+
+; SSRA
+
+define <vscale x 16 x i8> @ssra_disjoint_shift_intr_or16xi8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_intr_or16xi8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.b, z1.b, #7
+; CHECK-NEXT: lsr z0.b, z0.b, #2
+; CHECK-NEXT: asr z1.b, z1.b, #1
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+ %acc = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 2))
+ %sign = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> splat(i8 7))
+ %sra = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %sign, <vscale x 16 x i8> splat(i8 1))
+ %r = or <vscale x 16 x i8> %acc, %sra
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 8 x i16> @ssra_disjoint_shift_intr_or8xi16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_intr_or8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.h, z1.h, #15
+; CHECK-NEXT: lsr z0.h, z0.h, #10
+; CHECK-NEXT: asr z1.h, z1.h, #9
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+ %acc = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 10))
+ %sign = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> splat(i16 15))
+ %sra = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %sign, <vscale x 8 x i16> splat(i16 9))
+ %r = or <vscale x 8 x i16> %acc, %sra
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 4 x i32> @ssra_disjoint_shift_intr_or4xi32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_intr_or4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: lsr z0.s, z0.s, #26
+; CHECK-NEXT: asr z1.s, z1.s, #25
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+ %acc = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 26))
+ %sign = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> splat(i32 31))
+ %sra = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %sign, <vscale x 4 x i32> splat(i32 25))
+ %r = or <vscale x 4 x i32> %acc, %sra
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 2 x i64> @ssra_disjoint_shift_intr_or2xi64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+; CHECK-LABEL: ssra_disjoint_shift_intr_or2xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.d, z1.d, #63
+; CHECK-NEXT: lsr z0.d, z0.d, #58
+; CHECK-NEXT: asr z1.d, z1.d, #57
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+ %acc = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 58))
+ %sign = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> splat(i64 63))
+ %sra = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %sign, <vscale x 2 x i64> splat(i64 57))
+ %r = or <vscale x 2 x i64> %acc, %sra
+ ret <vscale x 2 x i64> %r
+}
+
declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 immarg)
declare <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 immarg)
declare <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 immarg)
@@ -345,6 +585,11 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1>, <v
declare <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.lsl.u.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.lsl.u.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.lsl.u.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.lsl.u.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
declare <vscale x 16 x i8> @llvm.aarch64.sve.asr.u.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 8 x i16> @llvm.aarch64.sve.asr.u.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.asr.u.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
>From 070240ce7e614e3893df94dbeb85a98719ab84b4 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Thu, 28 May 2026 11:22:24 +0000
Subject: [PATCH 2/2] fixup! [AArch64][SVE] Enable known bits for predicated
shifts
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 5 -
.../Target/AArch64/AArch64ISelLowering.cpp | 27 +-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 +-
.../CodeGen/AArch64/sve2-fixed-length-sra.ll | 761 ++++++++----------
llvm/test/CodeGen/AArch64/sve2-sli-sri.ll | 24 +-
llvm/test/CodeGen/AArch64/sve2-sra.ll | 64 +-
6 files changed, 394 insertions(+), 489 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index d632ac278860f..ae0b0992048ea 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4527,11 +4527,6 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
case ISD::INTRINSIC_WO_CHAIN:
case ISD::INTRINSIC_W_CHAIN:
case ISD::INTRINSIC_VOID:
- // TODO: Probably okay to remove after audit; here to reduce change size
- // in initial enablement patch for scalable vectors
- if (Op.getValueType().isScalableVector())
- break;
-
// Allow the target to implement this method for its nodes.
TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
break;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 57a2d73e00f57..5544b79bd5d28 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2937,6 +2937,26 @@ void AArch64TargetLowering::computeKnownBitsForTargetNode(
}
break;
}
+ case AArch64ISD::SHL_PRED:
+ case AArch64ISD::SRL_PRED:
+ case AArch64ISD::SRA_PRED: {
+ SDValue Pg = Op->getOperand(0);
+ if (!isAllActivePredicate(DAG, Pg) && (Pg.getOpcode() != AArch64ISD::PTRUE))
+ break;
+
+ KnownBits KnownVal =
+ DAG.computeKnownBits(Op->getOperand(1), DemandedElts, Depth + 1);
+ KnownBits KnownAmt =
+ DAG.computeKnownBits(Op->getOperand(2), DemandedElts, Depth + 1);
+
+ if (Op.getOpcode() == AArch64ISD::SHL_PRED)
+ Known = KnownBits::shl(KnownVal, KnownAmt);
+ else if (Op.getOpcode() == AArch64ISD::SRL_PRED)
+ Known = KnownBits::lshr(KnownVal, KnownAmt);
+ else
+ Known = KnownBits::ashr(KnownVal, KnownAmt);
+ break;
+ }
case ISD::INTRINSIC_WO_CHAIN:
case ISD::INTRINSIC_VOID: {
unsigned IntNo = Op.getConstantOperandVal(0);
@@ -15803,7 +15823,7 @@ static bool isAllInactivePredicate(SDValue N) {
return ISD::isConstantSplatVectorAllZeros(N.getNode());
}
-static bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) {
+static bool isAllActivePredicate(const SelectionDAG &DAG, SDValue N) {
unsigned NumElts = N.getValueType().getVectorMinNumElements();
// Look through cast.
@@ -32466,7 +32486,8 @@ SDValue AArch64TargetLowering::LowerToScalableOp(SDValue Op,
Ops.push_back(convertToScalableVector(DAG, ContainerVT, V));
}
- auto ScalableRes = DAG.getNode(Op.getOpcode(), SDLoc(Op), ContainerVT, Ops);
+ auto ScalableRes =
+ DAG.getNode(Op.getOpcode(), SDLoc(Op), ContainerVT, Ops, Op->getFlags());
return convertFromScalableVector(DAG, VT, ScalableRes);
}
@@ -33696,7 +33717,7 @@ SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
return Op;
}
-bool AArch64TargetLowering::isAllActivePredicate(SelectionDAG &DAG,
+bool AArch64TargetLowering::isAllActivePredicate(const SelectionDAG &DAG,
SDValue N) const {
return ::isAllActivePredicate(DAG, N);
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 7e4c4e1ba25ff..5c3e2e09d596c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -545,7 +545,7 @@ class AArch64TargetLowering : public TargetLowering {
return 128;
}
- bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const;
+ bool isAllActivePredicate(const SelectionDAG &DAG, SDValue N) const;
EVT getPromotedVTForPredicate(EVT VT) const;
EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty,
diff --git a/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll b/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll
index 891724998a913..1d83e1be1b2f0 100644
--- a/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll
@@ -33,10 +33,9 @@ define void @usra_disjoint_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: usra_disjoint_or32xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.b, z0.b, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: usra z0.b, z1.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i8>, ptr %a
@@ -52,25 +51,22 @@ define void @usra_disjoint_or64xi8(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.b, vl32
; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
-; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
-; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
-; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x0]
-; VBITS_GE_256-NEXT: lsr z0.b, z0.b, #4
-; VBITS_GE_256-NEXT: lsr z1.b, z1.b, #4
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT: usra z0.b, z1.b, #4
+; VBITS_GE_256-NEXT: usra z2.b, z3.b, #4
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
-; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; VBITS_GE_256-NEXT: st1b { z2.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: usra_disjoint_or64xi8:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.b, vl64
-; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x0]
-; VBITS_GE_512-NEXT: lsr z0.b, z0.b, #4
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: usra z0.b, z1.b, #4
; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <64 x i8>, ptr %a
@@ -85,10 +81,9 @@ define void @usra_disjoint_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: usra_disjoint_or128xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl128
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.b, z0.b, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: usra z0.b, z1.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i8>, ptr %a
@@ -103,10 +98,9 @@ define void @usra_disjoint_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0 {
; CHECK-LABEL: usra_disjoint_or256xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl256
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.b, z0.b, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: usra z0.b, z1.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <256 x i8>, ptr %a
@@ -141,10 +135,9 @@ define void @usra_disjoint_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: usra_disjoint_or16xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl16
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.h, z0.h, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: usra z0.h, z1.h, #4
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <16 x i16>, ptr %a
@@ -160,25 +153,22 @@ define void @usra_disjoint_or32xi16(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.h, vl16
; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
-; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1, x8, lsl #1]
-; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0]
-; VBITS_GE_256-NEXT: lsr z0.h, z0.h, #4
-; VBITS_GE_256-NEXT: lsr z1.h, z1.h, #4
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: usra z0.h, z1.h, #4
+; VBITS_GE_256-NEXT: usra z2.h, z3.h, #4
; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: usra_disjoint_or32xi16:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.h, vl32
-; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0]
-; VBITS_GE_512-NEXT: lsr z0.h, z0.h, #4
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: usra z0.h, z1.h, #4
; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <32 x i16>, ptr %a
@@ -193,10 +183,9 @@ define void @usra_disjoint_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: usra_disjoint_or64xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl64
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.h, z0.h, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: usra z0.h, z1.h, #4
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i16>, ptr %a
@@ -211,10 +200,9 @@ define void @usra_disjoint_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0 {
; CHECK-LABEL: usra_disjoint_or128xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl128
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.h, z0.h, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: usra z0.h, z1.h, #4
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i16>, ptr %a
@@ -249,10 +237,9 @@ define void @usra_disjoint_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: usra_disjoint_or8xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl8
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.s, z0.s, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: usra z0.s, z1.s, #4
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <8 x i32>, ptr %a
@@ -268,25 +255,22 @@ define void @usra_disjoint_or16xi32(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
-; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
-; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]
-; VBITS_GE_256-NEXT: lsr z0.s, z0.s, #4
-; VBITS_GE_256-NEXT: lsr z1.s, z1.s, #4
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: usra z0.s, z1.s, #4
+; VBITS_GE_256-NEXT: usra z2.s, z3.s, #4
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: usra_disjoint_or16xi32:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
-; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0]
-; VBITS_GE_512-NEXT: lsr z0.s, z0.s, #4
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: usra z0.s, z1.s, #4
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <16 x i32>, ptr %a
@@ -301,10 +285,9 @@ define void @usra_disjoint_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: usra_disjoint_or32xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl32
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.s, z0.s, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: usra z0.s, z1.s, #4
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i32>, ptr %a
@@ -319,10 +302,9 @@ define void @usra_disjoint_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0 {
; CHECK-LABEL: usra_disjoint_or64xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl64
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.s, z0.s, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: usra z0.s, z1.s, #4
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i32>, ptr %a
@@ -357,10 +339,9 @@ define void @usra_disjoint_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: usra_disjoint_or4xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.d, z0.d, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: usra z0.d, z1.d, #4
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <4 x i64>, ptr %a
@@ -376,25 +357,22 @@ define void @usra_disjoint_or8xi64(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
-; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
-; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]
-; VBITS_GE_256-NEXT: lsr z0.d, z0.d, #4
-; VBITS_GE_256-NEXT: lsr z1.d, z1.d, #4
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: usra z0.d, z1.d, #4
+; VBITS_GE_256-NEXT: usra z2.d, z3.d, #4
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
-; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: usra_disjoint_or8xi64:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
-; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0]
-; VBITS_GE_512-NEXT: lsr z0.d, z0.d, #4
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: usra z0.d, z1.d, #4
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <8 x i64>, ptr %a
@@ -409,10 +387,9 @@ define void @usra_disjoint_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: usra_disjoint_or16xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl16
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.d, z0.d, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: usra z0.d, z1.d, #4
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <16 x i64>, ptr %a
@@ -427,10 +404,9 @@ define void @usra_disjoint_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0 {
; CHECK-LABEL: usra_disjoint_or32xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl32
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.d, z0.d, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: usra z0.d, z1.d, #4
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i64>, ptr %a
@@ -449,10 +425,9 @@ define void @usra2_disjoint_or32i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: usra2_disjoint_or32i8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.b, z0.b, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: usra z0.b, z1.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i8>, ptr %a, align 32
@@ -468,16 +443,14 @@ define void @usra2_disjoint_or64i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl32
; CHECK-NEXT: mov w8, #32 // =0x20
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
-; CHECK-NEXT: ld1b { z3.b }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.b, z0.b, #4
-; CHECK-NEXT: lsr z1.b, z1.b, #4
-; CHECK-NEXT: orr z0.d, z2.d, z0.d
-; CHECK-NEXT: orr z1.d, z3.d, z1.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1, x8]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1]
+; CHECK-NEXT: usra z0.b, z1.b, #4
+; CHECK-NEXT: usra z2.b, z3.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0, x8]
-; CHECK-NEXT: st1b { z1.b }, p0, [x0]
+; CHECK-NEXT: st1b { z2.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i8>, ptr %a, align 32
%vb = load <64 x i8>, ptr %b, align 32
@@ -494,26 +467,22 @@ define void @usra2_disjoint_or128i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: mov w8, #64 // =0x40
; CHECK-NEXT: mov w9, #96 // =0x60
; CHECK-NEXT: mov w10, #32 // =0x20
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1, x9]
-; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x10]
-; CHECK-NEXT: ld1b { z4.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z3.b }, p0/z, [x0, x8]
-; CHECK-NEXT: ld1b { z5.b }, p0/z, [x0, x9]
-; CHECK-NEXT: ld1b { z6.b }, p0/z, [x0, x10]
-; CHECK-NEXT: ld1b { z7.b }, p0/z, [x0]
-; CHECK-NEXT: lsr z0.b, z0.b, #4
-; CHECK-NEXT: lsr z1.b, z1.b, #4
-; CHECK-NEXT: lsr z2.b, z2.b, #4
-; CHECK-NEXT: lsr z4.b, z4.b, #4
-; CHECK-NEXT: orr z0.d, z3.d, z0.d
-; CHECK-NEXT: orr z1.d, z5.d, z1.d
-; CHECK-NEXT: orr z2.d, z6.d, z2.d
-; CHECK-NEXT: orr z3.d, z7.d, z4.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1, x8]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x9]
+; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1, x9]
+; CHECK-NEXT: ld1b { z4.b }, p0/z, [x0, x10]
+; CHECK-NEXT: ld1b { z5.b }, p0/z, [x1, x10]
+; CHECK-NEXT: ld1b { z6.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z7.b }, p0/z, [x1]
+; CHECK-NEXT: usra z0.b, z1.b, #4
+; CHECK-NEXT: usra z2.b, z3.b, #4
+; CHECK-NEXT: usra z4.b, z5.b, #4
+; CHECK-NEXT: usra z6.b, z7.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0, x8]
-; CHECK-NEXT: st1b { z1.b }, p0, [x0, x9]
-; CHECK-NEXT: st1b { z2.b }, p0, [x0, x10]
-; CHECK-NEXT: st1b { z3.b }, p0, [x0]
+; CHECK-NEXT: st1b { z2.b }, p0, [x0, x9]
+; CHECK-NEXT: st1b { z4.b }, p0, [x0, x10]
+; CHECK-NEXT: st1b { z6.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i8>, ptr %a, align 32
%vb = load <128 x i8>, ptr %b, align 32
@@ -551,10 +520,9 @@ define void @ssra_disjoint_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: ssra_disjoint_or32xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: asr z0.b, z0.b, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.b, z1.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i8>, ptr %a
@@ -570,25 +538,22 @@ define void @ssra_disjoint_or64xi8(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.b, vl32
; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
-; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
-; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
-; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x0]
-; VBITS_GE_256-NEXT: asr z0.b, z0.b, #4
-; VBITS_GE_256-NEXT: asr z1.b, z1.b, #4
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ssra z0.b, z1.b, #4
+; VBITS_GE_256-NEXT: ssra z2.b, z3.b, #4
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
-; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; VBITS_GE_256-NEXT: st1b { z2.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: ssra_disjoint_or64xi8:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.b, vl64
-; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x0]
-; VBITS_GE_512-NEXT: asr z0.b, z0.b, #4
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ssra z0.b, z1.b, #4
; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <64 x i8>, ptr %a
@@ -603,10 +568,9 @@ define void @ssra_disjoint_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: ssra_disjoint_or128xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl128
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: asr z0.b, z0.b, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.b, z1.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i8>, ptr %a
@@ -621,10 +585,9 @@ define void @ssra_disjoint_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0 {
; CHECK-LABEL: ssra_disjoint_or256xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl256
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: asr z0.b, z0.b, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.b, z1.b, #4
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <256 x i8>, ptr %a
@@ -659,10 +622,9 @@ define void @ssra_disjoint_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: ssra_disjoint_or16xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl16
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: asr z0.h, z0.h, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.h, z1.h, #4
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <16 x i16>, ptr %a
@@ -678,25 +640,22 @@ define void @ssra_disjoint_or32xi16(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.h, vl16
; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
-; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1, x8, lsl #1]
-; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0]
-; VBITS_GE_256-NEXT: asr z0.h, z0.h, #4
-; VBITS_GE_256-NEXT: asr z1.h, z1.h, #4
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ssra z0.h, z1.h, #4
+; VBITS_GE_256-NEXT: ssra z2.h, z3.h, #4
; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: ssra_disjoint_or32xi16:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.h, vl32
-; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0]
-; VBITS_GE_512-NEXT: asr z0.h, z0.h, #4
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ssra z0.h, z1.h, #4
; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <32 x i16>, ptr %a
@@ -711,10 +670,9 @@ define void @ssra_disjoint_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: ssra_disjoint_or64xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl64
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: asr z0.h, z0.h, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.h, z1.h, #4
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i16>, ptr %a
@@ -729,10 +687,9 @@ define void @ssra_disjoint_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0 {
; CHECK-LABEL: ssra_disjoint_or128xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl128
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: asr z0.h, z0.h, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.h, z1.h, #4
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i16>, ptr %a
@@ -767,10 +724,9 @@ define void @ssra_disjoint_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: ssra_disjoint_or8xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl8
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: asr z0.s, z0.s, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.s, z1.s, #4
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <8 x i32>, ptr %a
@@ -786,25 +742,22 @@ define void @ssra_disjoint_or16xi32(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
-; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
-; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]
-; VBITS_GE_256-NEXT: asr z0.s, z0.s, #4
-; VBITS_GE_256-NEXT: asr z1.s, z1.s, #4
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ssra z0.s, z1.s, #4
+; VBITS_GE_256-NEXT: ssra z2.s, z3.s, #4
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: ssra_disjoint_or16xi32:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
-; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0]
-; VBITS_GE_512-NEXT: asr z0.s, z0.s, #4
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ssra z0.s, z1.s, #4
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <16 x i32>, ptr %a
@@ -819,10 +772,9 @@ define void @ssra_disjoint_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: ssra_disjoint_or32xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl32
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: asr z0.s, z0.s, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.s, z1.s, #4
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i32>, ptr %a
@@ -837,10 +789,9 @@ define void @ssra_disjoint_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0 {
; CHECK-LABEL: ssra_disjoint_or64xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl64
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: asr z0.s, z0.s, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.s, z1.s, #4
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i32>, ptr %a
@@ -875,10 +826,9 @@ define void @ssra_disjoint_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: ssra_disjoint_or4xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: asr z0.d, z0.d, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.d, z1.d, #4
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <4 x i64>, ptr %a
@@ -894,25 +844,22 @@ define void @ssra_disjoint_or8xi64(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
-; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
-; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]
-; VBITS_GE_256-NEXT: asr z0.d, z0.d, #4
-; VBITS_GE_256-NEXT: asr z1.d, z1.d, #4
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: ssra z0.d, z1.d, #4
+; VBITS_GE_256-NEXT: ssra z2.d, z3.d, #4
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
-; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: ssra_disjoint_or8xi64:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
-; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0]
-; VBITS_GE_512-NEXT: asr z0.d, z0.d, #4
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: ssra z0.d, z1.d, #4
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <8 x i64>, ptr %a
@@ -927,10 +874,9 @@ define void @ssra_disjoint_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: ssra_disjoint_or16xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl16
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: asr z0.d, z0.d, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.d, z1.d, #4
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <16 x i64>, ptr %a
@@ -945,10 +891,9 @@ define void @ssra_disjoint_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0 {
; CHECK-LABEL: ssra_disjoint_or32xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl32
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: asr z0.d, z0.d, #4
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: ssra z0.d, z1.d, #4
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i64>, ptr %a
@@ -994,8 +939,7 @@ define void @usra_disjoint_shift_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
; CHECK-NEXT: lsl z0.b, z0.b, #7
-; CHECK-NEXT: lsr z1.b, z1.b, #1
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.b, z1.b, #1
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i8>, ptr %a
@@ -1013,15 +957,13 @@ define void @usra_disjoint_shift_or64xi8(ptr %a, ptr %b) #0 {
; VBITS_GE_256-NEXT: ptrue p0.b, vl32
; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
-; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]
-; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x1, x8]
; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]
; VBITS_GE_256-NEXT: lsl z0.b, z0.b, #7
-; VBITS_GE_256-NEXT: lsr z1.b, z1.b, #1
-; VBITS_GE_256-NEXT: lsl z2.b, z2.b, #7
-; VBITS_GE_256-NEXT: lsr z3.b, z3.b, #1
-; VBITS_GE_256-NEXT: orr z0.d, z0.d, z1.d
-; VBITS_GE_256-NEXT: orr z1.d, z2.d, z3.d
+; VBITS_GE_256-NEXT: lsl z1.b, z1.b, #7
+; VBITS_GE_256-NEXT: usra z0.b, z2.b, #1
+; VBITS_GE_256-NEXT: usra z1.b, z3.b, #1
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
@@ -1032,8 +974,7 @@ define void @usra_disjoint_shift_or64xi8(ptr %a, ptr %b) #0 {
; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]
; VBITS_GE_512-NEXT: lsl z0.b, z0.b, #7
-; VBITS_GE_512-NEXT: lsr z1.b, z1.b, #1
-; VBITS_GE_512-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_512-NEXT: usra z0.b, z1.b, #1
; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <64 x i8>, ptr %a
@@ -1052,8 +993,7 @@ define void @usra_disjoint_shift_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
; CHECK-NEXT: lsl z0.b, z0.b, #7
-; CHECK-NEXT: lsr z1.b, z1.b, #1
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.b, z1.b, #1
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i8>, ptr %a
@@ -1072,8 +1012,7 @@ define void @usra_disjoint_shift_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0
; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
; CHECK-NEXT: lsl z0.b, z0.b, #7
-; CHECK-NEXT: lsr z1.b, z1.b, #1
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.b, z1.b, #1
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <256 x i8>, ptr %a
@@ -1116,8 +1055,7 @@ define void @usra_disjoint_shift_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
; CHECK-NEXT: lsl z0.h, z0.h, #7
-; CHECK-NEXT: lsr z1.h, z1.h, #9
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.h, z1.h, #9
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <16 x i16>, ptr %a
@@ -1135,15 +1073,13 @@ define void @usra_disjoint_shift_or32xi16(ptr %a, ptr %b) #0 {
; VBITS_GE_256-NEXT: ptrue p0.h, vl16
; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
-; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1]
; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]
; VBITS_GE_256-NEXT: lsl z0.h, z0.h, #7
-; VBITS_GE_256-NEXT: lsr z1.h, z1.h, #9
-; VBITS_GE_256-NEXT: lsl z2.h, z2.h, #7
-; VBITS_GE_256-NEXT: lsr z3.h, z3.h, #9
-; VBITS_GE_256-NEXT: orr z0.d, z0.d, z1.d
-; VBITS_GE_256-NEXT: orr z1.d, z2.d, z3.d
+; VBITS_GE_256-NEXT: lsl z1.h, z1.h, #7
+; VBITS_GE_256-NEXT: usra z0.h, z2.h, #9
+; VBITS_GE_256-NEXT: usra z1.h, z3.h, #9
; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
; VBITS_GE_256-NEXT: ret
@@ -1154,8 +1090,7 @@ define void @usra_disjoint_shift_or32xi16(ptr %a, ptr %b) #0 {
; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
; VBITS_GE_512-NEXT: lsl z0.h, z0.h, #7
-; VBITS_GE_512-NEXT: lsr z1.h, z1.h, #9
-; VBITS_GE_512-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_512-NEXT: usra z0.h, z1.h, #9
; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <32 x i16>, ptr %a
@@ -1174,8 +1109,7 @@ define void @usra_disjoint_shift_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
; CHECK-NEXT: lsl z0.h, z0.h, #7
-; CHECK-NEXT: lsr z1.h, z1.h, #9
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.h, z1.h, #9
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i16>, ptr %a
@@ -1194,8 +1128,7 @@ define void @usra_disjoint_shift_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0
; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
; CHECK-NEXT: lsl z0.h, z0.h, #7
-; CHECK-NEXT: lsr z1.h, z1.h, #9
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.h, z1.h, #9
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i16>, ptr %a
@@ -1238,8 +1171,7 @@ define void @usra_disjoint_shift_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
; CHECK-NEXT: lsl z0.s, z0.s, #7
-; CHECK-NEXT: lsr z1.s, z1.s, #25
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.s, z1.s, #25
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <8 x i32>, ptr %a
@@ -1257,15 +1189,13 @@ define void @usra_disjoint_shift_or16xi32(ptr %a, ptr %b) #0 {
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
-; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]
; VBITS_GE_256-NEXT: lsl z0.s, z0.s, #7
-; VBITS_GE_256-NEXT: lsr z1.s, z1.s, #25
-; VBITS_GE_256-NEXT: lsl z2.s, z2.s, #7
-; VBITS_GE_256-NEXT: lsr z3.s, z3.s, #25
-; VBITS_GE_256-NEXT: orr z0.d, z0.d, z1.d
-; VBITS_GE_256-NEXT: orr z1.d, z2.d, z3.d
+; VBITS_GE_256-NEXT: lsl z1.s, z1.s, #7
+; VBITS_GE_256-NEXT: usra z0.s, z2.s, #25
+; VBITS_GE_256-NEXT: usra z1.s, z3.s, #25
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
; VBITS_GE_256-NEXT: ret
@@ -1276,8 +1206,7 @@ define void @usra_disjoint_shift_or16xi32(ptr %a, ptr %b) #0 {
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
; VBITS_GE_512-NEXT: lsl z0.s, z0.s, #7
-; VBITS_GE_512-NEXT: lsr z1.s, z1.s, #25
-; VBITS_GE_512-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_512-NEXT: usra z0.s, z1.s, #25
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <16 x i32>, ptr %a
@@ -1296,8 +1225,7 @@ define void @usra_disjoint_shift_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
; CHECK-NEXT: lsl z0.s, z0.s, #7
-; CHECK-NEXT: lsr z1.s, z1.s, #25
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.s, z1.s, #25
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i32>, ptr %a
@@ -1316,8 +1244,7 @@ define void @usra_disjoint_shift_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
; CHECK-NEXT: lsl z0.s, z0.s, #7
-; CHECK-NEXT: lsr z1.s, z1.s, #25
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.s, z1.s, #25
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i32>, ptr %a
@@ -1360,8 +1287,7 @@ define void @usra_disjoint_shift_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: lsl z0.d, z0.d, #7
-; CHECK-NEXT: lsr z1.d, z1.d, #57
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.d, z1.d, #57
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <4 x i64>, ptr %a
@@ -1379,15 +1305,13 @@ define void @usra_disjoint_shift_or8xi64(ptr %a, ptr %b) #0 {
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]
; VBITS_GE_256-NEXT: lsl z0.d, z0.d, #7
-; VBITS_GE_256-NEXT: lsr z1.d, z1.d, #57
-; VBITS_GE_256-NEXT: lsl z2.d, z2.d, #7
-; VBITS_GE_256-NEXT: lsr z3.d, z3.d, #57
-; VBITS_GE_256-NEXT: orr z0.d, z0.d, z1.d
-; VBITS_GE_256-NEXT: orr z1.d, z2.d, z3.d
+; VBITS_GE_256-NEXT: lsl z1.d, z1.d, #7
+; VBITS_GE_256-NEXT: usra z0.d, z2.d, #57
+; VBITS_GE_256-NEXT: usra z1.d, z3.d, #57
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
; VBITS_GE_256-NEXT: ret
@@ -1398,8 +1322,7 @@ define void @usra_disjoint_shift_or8xi64(ptr %a, ptr %b) #0 {
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: lsl z0.d, z0.d, #7
-; VBITS_GE_512-NEXT: lsr z1.d, z1.d, #57
-; VBITS_GE_512-NEXT: orr z0.d, z0.d, z1.d
+; VBITS_GE_512-NEXT: usra z0.d, z1.d, #57
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <8 x i64>, ptr %a
@@ -1418,8 +1341,7 @@ define void @usra_disjoint_shift_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: lsl z0.d, z0.d, #7
-; CHECK-NEXT: lsr z1.d, z1.d, #57
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.d, z1.d, #57
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <16 x i64>, ptr %a
@@ -1438,8 +1360,7 @@ define void @usra_disjoint_shift_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: lsl z0.d, z0.d, #7
-; CHECK-NEXT: lsr z1.d, z1.d, #57
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.d, z1.d, #57
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i64>, ptr %a
@@ -1487,12 +1408,11 @@ define void @ssra_disjoint_shift_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or32xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.b, z0.b, #7
-; CHECK-NEXT: lsr z1.b, z1.b, #2
-; CHECK-NEXT: asr z0.b, z0.b, #1
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.b, z0.b, #2
+; CHECK-NEXT: lsl z1.b, z1.b, #7
+; CHECK-NEXT: ssra z0.b, z1.b, #1
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i8>, ptr %a
@@ -1510,31 +1430,28 @@ define void @ssra_disjoint_shift_or64xi8(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.b, vl32
; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
-; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1, x8]
-; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
-; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x0]
-; VBITS_GE_256-NEXT: lsl z0.b, z0.b, #7
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT: lsr z0.b, z0.b, #2
; VBITS_GE_256-NEXT: lsl z1.b, z1.b, #7
; VBITS_GE_256-NEXT: lsr z2.b, z2.b, #2
-; VBITS_GE_256-NEXT: lsr z3.b, z3.b, #2
-; VBITS_GE_256-NEXT: asr z0.b, z0.b, #1
-; VBITS_GE_256-NEXT: asr z1.b, z1.b, #1
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: lsl z3.b, z3.b, #7
+; VBITS_GE_256-NEXT: ssra z0.b, z1.b, #1
+; VBITS_GE_256-NEXT: ssra z2.b, z3.b, #1
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
-; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; VBITS_GE_256-NEXT: st1b { z2.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: ssra_disjoint_shift_or64xi8:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.b, vl64
-; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x0]
-; VBITS_GE_512-NEXT: lsl z0.b, z0.b, #7
-; VBITS_GE_512-NEXT: lsr z1.b, z1.b, #2
-; VBITS_GE_512-NEXT: asr z0.b, z0.b, #1
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: lsr z0.b, z0.b, #2
+; VBITS_GE_512-NEXT: lsl z1.b, z1.b, #7
+; VBITS_GE_512-NEXT: ssra z0.b, z1.b, #1
; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <64 x i8>, ptr %a
@@ -1551,12 +1468,11 @@ define void @ssra_disjoint_shift_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or128xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl128
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.b, z0.b, #7
-; CHECK-NEXT: lsr z1.b, z1.b, #2
-; CHECK-NEXT: asr z0.b, z0.b, #1
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.b, z0.b, #2
+; CHECK-NEXT: lsl z1.b, z1.b, #7
+; CHECK-NEXT: ssra z0.b, z1.b, #1
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i8>, ptr %a
@@ -1573,12 +1489,11 @@ define void @ssra_disjoint_shift_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0
; CHECK-LABEL: ssra_disjoint_shift_or256xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl256
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.b, z0.b, #7
-; CHECK-NEXT: lsr z1.b, z1.b, #2
-; CHECK-NEXT: asr z0.b, z0.b, #1
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.b, z0.b, #2
+; CHECK-NEXT: lsl z1.b, z1.b, #7
+; CHECK-NEXT: ssra z0.b, z1.b, #1
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%va = load <256 x i8>, ptr %a
@@ -1623,12 +1538,11 @@ define void @ssra_disjoint_shift_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or16xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl16
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.h, z0.h, #15
-; CHECK-NEXT: lsr z1.h, z1.h, #10
-; CHECK-NEXT: asr z0.h, z0.h, #9
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.h, z0.h, #10
+; CHECK-NEXT: lsl z1.h, z1.h, #15
+; CHECK-NEXT: ssra z0.h, z1.h, #9
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <16 x i16>, ptr %a
@@ -1646,31 +1560,28 @@ define void @ssra_disjoint_shift_or32xi16(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.h, vl16
; VBITS_GE_256-NEXT: mov x8, #16 // =0x10
-; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1, x8, lsl #1]
-; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0]
-; VBITS_GE_256-NEXT: lsl z0.h, z0.h, #15
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: lsr z0.h, z0.h, #10
; VBITS_GE_256-NEXT: lsl z1.h, z1.h, #15
; VBITS_GE_256-NEXT: lsr z2.h, z2.h, #10
-; VBITS_GE_256-NEXT: lsr z3.h, z3.h, #10
-; VBITS_GE_256-NEXT: asr z0.h, z0.h, #9
-; VBITS_GE_256-NEXT: asr z1.h, z1.h, #9
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: lsl z3.h, z3.h, #15
+; VBITS_GE_256-NEXT: ssra z0.h, z1.h, #9
+; VBITS_GE_256-NEXT: ssra z2.h, z3.h, #9
; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: ssra_disjoint_shift_or32xi16:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.h, vl32
-; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0]
-; VBITS_GE_512-NEXT: lsl z0.h, z0.h, #15
-; VBITS_GE_512-NEXT: lsr z1.h, z1.h, #10
-; VBITS_GE_512-NEXT: asr z0.h, z0.h, #9
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: lsr z0.h, z0.h, #10
+; VBITS_GE_512-NEXT: lsl z1.h, z1.h, #15
+; VBITS_GE_512-NEXT: ssra z0.h, z1.h, #9
; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <32 x i16>, ptr %a
@@ -1687,12 +1598,11 @@ define void @ssra_disjoint_shift_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or64xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl64
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.h, z0.h, #15
-; CHECK-NEXT: lsr z1.h, z1.h, #10
-; CHECK-NEXT: asr z0.h, z0.h, #9
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.h, z0.h, #10
+; CHECK-NEXT: lsl z1.h, z1.h, #15
+; CHECK-NEXT: ssra z0.h, z1.h, #9
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i16>, ptr %a
@@ -1709,12 +1619,11 @@ define void @ssra_disjoint_shift_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0
; CHECK-LABEL: ssra_disjoint_shift_or128xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl128
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x1]
-; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.h, z0.h, #15
-; CHECK-NEXT: lsr z1.h, z1.h, #10
-; CHECK-NEXT: asr z0.h, z0.h, #9
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.h, z0.h, #10
+; CHECK-NEXT: lsl z1.h, z1.h, #15
+; CHECK-NEXT: ssra z0.h, z1.h, #9
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%va = load <128 x i16>, ptr %a
@@ -1759,12 +1668,11 @@ define void @ssra_disjoint_shift_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or8xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl8
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.s, z0.s, #31
-; CHECK-NEXT: lsr z1.s, z1.s, #26
-; CHECK-NEXT: asr z0.s, z0.s, #25
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.s, z0.s, #26
+; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: ssra z0.s, z1.s, #25
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <8 x i32>, ptr %a
@@ -1782,31 +1690,28 @@ define void @ssra_disjoint_shift_or16xi32(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
-; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
-; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]
-; VBITS_GE_256-NEXT: lsl z0.s, z0.s, #31
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: lsr z0.s, z0.s, #26
; VBITS_GE_256-NEXT: lsl z1.s, z1.s, #31
; VBITS_GE_256-NEXT: lsr z2.s, z2.s, #26
-; VBITS_GE_256-NEXT: lsr z3.s, z3.s, #26
-; VBITS_GE_256-NEXT: asr z0.s, z0.s, #25
-; VBITS_GE_256-NEXT: asr z1.s, z1.s, #25
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: lsl z3.s, z3.s, #31
+; VBITS_GE_256-NEXT: ssra z0.s, z1.s, #25
+; VBITS_GE_256-NEXT: ssra z2.s, z3.s, #25
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: ssra_disjoint_shift_or16xi32:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
-; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0]
-; VBITS_GE_512-NEXT: lsl z0.s, z0.s, #31
-; VBITS_GE_512-NEXT: lsr z1.s, z1.s, #26
-; VBITS_GE_512-NEXT: asr z0.s, z0.s, #25
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: lsr z0.s, z0.s, #26
+; VBITS_GE_512-NEXT: lsl z1.s, z1.s, #31
+; VBITS_GE_512-NEXT: ssra z0.s, z1.s, #25
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <16 x i32>, ptr %a
@@ -1823,12 +1728,11 @@ define void @ssra_disjoint_shift_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or32xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl32
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.s, z0.s, #31
-; CHECK-NEXT: lsr z1.s, z1.s, #26
-; CHECK-NEXT: asr z0.s, z0.s, #25
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.s, z0.s, #26
+; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: ssra z0.s, z1.s, #25
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i32>, ptr %a
@@ -1845,12 +1749,11 @@ define void @ssra_disjoint_shift_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0
; CHECK-LABEL: ssra_disjoint_shift_or64xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl64
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.s, z0.s, #31
-; CHECK-NEXT: lsr z1.s, z1.s, #26
-; CHECK-NEXT: asr z0.s, z0.s, #25
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.s, z0.s, #26
+; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: ssra z0.s, z1.s, #25
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%va = load <64 x i32>, ptr %a
@@ -1895,12 +1798,11 @@ define void @ssra_disjoint_shift_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or4xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.d, z0.d, #63
-; CHECK-NEXT: lsr z1.d, z1.d, #58
-; CHECK-NEXT: asr z0.d, z0.d, #57
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.d, z0.d, #58
+; CHECK-NEXT: lsl z1.d, z1.d, #63
+; CHECK-NEXT: ssra z0.d, z1.d, #57
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <4 x i64>, ptr %a
@@ -1918,31 +1820,28 @@ define void @ssra_disjoint_shift_or8xi64(ptr %a, ptr %b) #0 {
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
-; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
-; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
-; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]
-; VBITS_GE_256-NEXT: lsl z0.d, z0.d, #63
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: lsr z0.d, z0.d, #58
; VBITS_GE_256-NEXT: lsl z1.d, z1.d, #63
; VBITS_GE_256-NEXT: lsr z2.d, z2.d, #58
-; VBITS_GE_256-NEXT: lsr z3.d, z3.d, #58
-; VBITS_GE_256-NEXT: asr z0.d, z0.d, #57
-; VBITS_GE_256-NEXT: asr z1.d, z1.d, #57
-; VBITS_GE_256-NEXT: orr z0.d, z2.d, z0.d
-; VBITS_GE_256-NEXT: orr z1.d, z3.d, z1.d
+; VBITS_GE_256-NEXT: lsl z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: ssra z0.d, z1.d, #57
+; VBITS_GE_256-NEXT: ssra z2.d, z3.d, #57
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
-; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; VBITS_GE_512-LABEL: ssra_disjoint_shift_or8xi64:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
-; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
-; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0]
-; VBITS_GE_512-NEXT: lsl z0.d, z0.d, #63
-; VBITS_GE_512-NEXT: lsr z1.d, z1.d, #58
-; VBITS_GE_512-NEXT: asr z0.d, z0.d, #57
-; VBITS_GE_512-NEXT: orr z0.d, z1.d, z0.d
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: lsr z0.d, z0.d, #58
+; VBITS_GE_512-NEXT: lsl z1.d, z1.d, #63
+; VBITS_GE_512-NEXT: ssra z0.d, z1.d, #57
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%va = load <8 x i64>, ptr %a
@@ -1959,12 +1858,11 @@ define void @ssra_disjoint_shift_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or16xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl16
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.d, z0.d, #63
-; CHECK-NEXT: lsr z1.d, z1.d, #58
-; CHECK-NEXT: asr z0.d, z0.d, #57
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.d, z0.d, #58
+; CHECK-NEXT: lsl z1.d, z1.d, #63
+; CHECK-NEXT: ssra z0.d, z1.d, #57
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <16 x i64>, ptr %a
@@ -1981,12 +1879,11 @@ define void @ssra_disjoint_shift_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0
; CHECK-LABEL: ssra_disjoint_shift_or32xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl32
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT: lsl z0.d, z0.d, #63
-; CHECK-NEXT: lsr z1.d, z1.d, #58
-; CHECK-NEXT: asr z0.d, z0.d, #57
-; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: lsr z0.d, z0.d, #58
+; CHECK-NEXT: lsl z1.d, z1.d, #63
+; CHECK-NEXT: ssra z0.d, z1.d, #57
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%va = load <32 x i64>, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve2-sli-sri.ll b/llvm/test/CodeGen/AArch64/sve2-sli-sri.ll
index 80999fb1f4864..a7048731850d3 100644
--- a/llvm/test/CodeGen/AArch64/sve2-sli-sri.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-sli-sri.ll
@@ -118,14 +118,22 @@ define <vscale x 8 x i16> @testRightGood8x16(<vscale x 8 x i16> %src1, <vscale x
}
define <vscale x 8 x i16> @testRightBad8x16(<vscale x 8 x i16> %src1, <vscale x 8 x i16> %src2) {
-; CHECK-LABEL: testRightBad8x16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #16500 // =0x4074
-; CHECK-NEXT: lsr z1.h, z1.h, #14
-; CHECK-NEXT: mov z2.h, w8
-; CHECK-NEXT: and z0.d, z0.d, z2.d
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; SVE-LABEL: testRightBad8x16:
+; SVE: // %bb.0:
+; SVE-NEXT: mov w8, #16500 // =0x4074
+; SVE-NEXT: lsr z1.h, z1.h, #14
+; SVE-NEXT: mov z2.h, w8
+; SVE-NEXT: and z0.d, z0.d, z2.d
+; SVE-NEXT: orr z0.d, z0.d, z1.d
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: testRightBad8x16:
+; SVE2: // %bb.0:
+; SVE2-NEXT: mov w8, #16500 // =0x4074
+; SVE2-NEXT: mov z2.h, w8
+; SVE2-NEXT: and z0.d, z0.d, z2.d
+; SVE2-NEXT: usra z0.h, z1.h, #14
+; SVE2-NEXT: ret
%and.i = and <vscale x 8 x i16> %src1, splat(i16 16500)
%vshl_n = lshr <vscale x 8 x i16> %src2, splat(i16 14)
%result = or <vscale x 8 x i16> %and.i, %vshl_n
diff --git a/llvm/test/CodeGen/AArch64/sve2-sra.ll b/llvm/test/CodeGen/AArch64/sve2-sra.ll
index 5e5f4234e5e06..e317af0b1de7b 100644
--- a/llvm/test/CodeGen/AArch64/sve2-sra.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-sra.ll
@@ -341,8 +341,7 @@ define <vscale x 16 x i8> @usra_disjoint_shift_or16xi8(<vscale x 16 x i8> %a, <v
; CHECK-LABEL: usra_disjoint_shift_or16xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: lsl z0.b, z0.b, #7
-; CHECK-NEXT: lsr z1.b, z1.b, #1
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.b, z1.b, #1
; CHECK-NEXT: ret
%shl = shl <vscale x 16 x i8> %a, splat (i8 7)
%srl = lshr <vscale x 16 x i8> %b, splat (i8 1)
@@ -354,8 +353,7 @@ define <vscale x 8 x i16> @usra_disjoint_shift_or8xi16(<vscale x 8 x i16> %a, <v
; CHECK-LABEL: usra_disjoint_shift_or8xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: lsl z0.h, z0.h, #7
-; CHECK-NEXT: lsr z1.h, z1.h, #9
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.h, z1.h, #9
; CHECK-NEXT: ret
%shl = shl <vscale x 8 x i16> %a, splat (i16 7)
%srl = lshr <vscale x 8 x i16> %b, splat (i16 9)
@@ -367,8 +365,7 @@ define <vscale x 4 x i32> @usra_disjoint_shift_or4xi32(<vscale x 4 x i32> %a, <v
; CHECK-LABEL: usra_disjoint_shift_or4xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: lsl z0.s, z0.s, #7
-; CHECK-NEXT: lsr z1.s, z1.s, #25
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.s, z1.s, #25
; CHECK-NEXT: ret
%shl = shl <vscale x 4 x i32> %a, splat (i32 7)
%srl = lshr <vscale x 4 x i32> %b, splat (i32 25)
@@ -380,8 +377,7 @@ define <vscale x 2 x i64> @usra_disjoint_shift_or2xi64(<vscale x 2 x i64> %a, <v
; CHECK-LABEL: usra_disjoint_shift_or2xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: lsl z0.d, z0.d, #7
-; CHECK-NEXT: lsr z1.d, z1.d, #57
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.d, z1.d, #57
; CHECK-NEXT: ret
%shl = shl <vscale x 2 x i64> %a, splat (i64 7)
%srl = lshr <vscale x 2 x i64> %b, splat (i64 57)
@@ -394,10 +390,9 @@ define <vscale x 2 x i64> @usra_disjoint_shift_or2xi64(<vscale x 2 x i64> %a, <v
define <vscale x 16 x i8> @ssra_disjoint_shift_or16xi8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or16xi8:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl z1.b, z1.b, #7
; CHECK-NEXT: lsr z0.b, z0.b, #2
-; CHECK-NEXT: asr z1.b, z1.b, #1
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: lsl z1.b, z1.b, #7
+; CHECK-NEXT: ssra z0.b, z1.b, #1
; CHECK-NEXT: ret
%acc = lshr <vscale x 16 x i8> %a, splat (i8 2)
%sign = shl <vscale x 16 x i8> %b, splat (i8 7)
@@ -409,10 +404,9 @@ define <vscale x 16 x i8> @ssra_disjoint_shift_or16xi8(<vscale x 16 x i8> %a, <v
define <vscale x 8 x i16> @ssra_disjoint_shift_or8xi16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or8xi16:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl z1.h, z1.h, #15
; CHECK-NEXT: lsr z0.h, z0.h, #10
-; CHECK-NEXT: asr z1.h, z1.h, #9
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: lsl z1.h, z1.h, #15
+; CHECK-NEXT: ssra z0.h, z1.h, #9
; CHECK-NEXT: ret
%acc = lshr <vscale x 8 x i16> %a, splat (i16 10)
%sign = shl <vscale x 8 x i16> %b, splat (i16 15)
@@ -424,10 +418,9 @@ define <vscale x 8 x i16> @ssra_disjoint_shift_or8xi16(<vscale x 8 x i16> %a, <v
define <vscale x 4 x i32> @ssra_disjoint_shift_or4xi32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or4xi32:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl z1.s, z1.s, #31
; CHECK-NEXT: lsr z0.s, z0.s, #26
-; CHECK-NEXT: asr z1.s, z1.s, #25
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: ssra z0.s, z1.s, #25
; CHECK-NEXT: ret
%acc = lshr <vscale x 4 x i32> %a, splat (i32 26)
%sign = shl <vscale x 4 x i32> %b, splat (i32 31)
@@ -439,10 +432,9 @@ define <vscale x 4 x i32> @ssra_disjoint_shift_or4xi32(<vscale x 4 x i32> %a, <v
define <vscale x 2 x i64> @ssra_disjoint_shift_or2xi64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
; CHECK-LABEL: ssra_disjoint_shift_or2xi64:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl z1.d, z1.d, #63
; CHECK-NEXT: lsr z0.d, z0.d, #58
-; CHECK-NEXT: asr z1.d, z1.d, #57
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: lsl z1.d, z1.d, #63
+; CHECK-NEXT: ssra z0.d, z1.d, #57
; CHECK-NEXT: ret
%acc = lshr <vscale x 2 x i64> %a, splat (i64 58)
%sign = shl <vscale x 2 x i64> %b, splat (i64 63)
@@ -457,8 +449,7 @@ define <vscale x 16 x i8> @usra_disjoint_shl_lsr_or16xi8(<vscale x 16 x i1> %pg,
; CHECK-LABEL: usra_disjoint_shl_lsr_or16xi8:
; CHECK: // %bb.0:
; CHECK-NEXT: lsl z0.b, z0.b, #4
-; CHECK-NEXT: lsr z1.b, z1.b, #4
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.b, z1.b, #4
; CHECK-NEXT: ret
%ptrue = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
%shl = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.u.nxv16i8(<vscale x 16 x i1> %ptrue, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 4))
@@ -471,8 +462,7 @@ define <vscale x 8 x i16> @usra_disjoint_shl_lsr_or8xi16(<vscale x 8 x i1> %pg,
; CHECK-LABEL: usra_disjoint_shl_lsr_or8xi16:
; CHECK: // %bb.0:
; CHECK-NEXT: lsl z0.h, z0.h, #8
-; CHECK-NEXT: lsr z1.h, z1.h, #8
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.h, z1.h, #8
; CHECK-NEXT: ret
%ptrue = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
%shl = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.u.nxv8i16(<vscale x 8 x i1> %ptrue, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 8))
@@ -485,8 +475,7 @@ define <vscale x 4 x i32> @usra_disjoint_shl_lsr_or4xi32(<vscale x 4 x i1> %pg,
; CHECK-LABEL: usra_disjoint_shl_lsr_or4xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: lsr z1.s, z1.s, #16
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.s, z1.s, #16
; CHECK-NEXT: ret
%ptrue = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
%shl = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.u.nxv4i32(<vscale x 4 x i1> %ptrue, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 16))
@@ -499,8 +488,7 @@ define <vscale x 2 x i64> @usra_disjoint_shl_lsr_or2xi64(<vscale x 2 x i1> %pg,
; CHECK-LABEL: usra_disjoint_shl_lsr_or2xi64:
; CHECK: // %bb.0:
; CHECK-NEXT: lsl z0.d, z0.d, #32
-; CHECK-NEXT: lsr z1.d, z1.d, #32
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: usra z0.d, z1.d, #32
; CHECK-NEXT: ret
%ptrue = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
%shl = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.u.nxv2i64(<vscale x 2 x i1> %ptrue, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 32))
@@ -514,10 +502,9 @@ define <vscale x 2 x i64> @usra_disjoint_shl_lsr_or2xi64(<vscale x 2 x i1> %pg,
define <vscale x 16 x i8> @ssra_disjoint_shift_intr_or16xi8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
; CHECK-LABEL: ssra_disjoint_shift_intr_or16xi8:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl z1.b, z1.b, #7
; CHECK-NEXT: lsr z0.b, z0.b, #2
-; CHECK-NEXT: asr z1.b, z1.b, #1
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: lsl z1.b, z1.b, #7
+; CHECK-NEXT: ssra z0.b, z1.b, #1
; CHECK-NEXT: ret
%pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
%acc = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 2))
@@ -530,10 +517,9 @@ define <vscale x 16 x i8> @ssra_disjoint_shift_intr_or16xi8(<vscale x 16 x i8> %
define <vscale x 8 x i16> @ssra_disjoint_shift_intr_or8xi16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
; CHECK-LABEL: ssra_disjoint_shift_intr_or8xi16:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl z1.h, z1.h, #15
; CHECK-NEXT: lsr z0.h, z0.h, #10
-; CHECK-NEXT: asr z1.h, z1.h, #9
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: lsl z1.h, z1.h, #15
+; CHECK-NEXT: ssra z0.h, z1.h, #9
; CHECK-NEXT: ret
%pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
%acc = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 10))
@@ -546,10 +532,9 @@ define <vscale x 8 x i16> @ssra_disjoint_shift_intr_or8xi16(<vscale x 8 x i16> %
define <vscale x 4 x i32> @ssra_disjoint_shift_intr_or4xi32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
; CHECK-LABEL: ssra_disjoint_shift_intr_or4xi32:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl z1.s, z1.s, #31
; CHECK-NEXT: lsr z0.s, z0.s, #26
-; CHECK-NEXT: asr z1.s, z1.s, #25
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: ssra z0.s, z1.s, #25
; CHECK-NEXT: ret
%pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
%acc = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 26))
@@ -562,10 +547,9 @@ define <vscale x 4 x i32> @ssra_disjoint_shift_intr_or4xi32(<vscale x 4 x i32> %
define <vscale x 2 x i64> @ssra_disjoint_shift_intr_or2xi64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
; CHECK-LABEL: ssra_disjoint_shift_intr_or2xi64:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl z1.d, z1.d, #63
; CHECK-NEXT: lsr z0.d, z0.d, #58
-; CHECK-NEXT: asr z1.d, z1.d, #57
-; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: lsl z1.d, z1.d, #63
+; CHECK-NEXT: ssra z0.d, z1.d, #57
; CHECK-NEXT: ret
%pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
%acc = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 58))
More information about the llvm-commits
mailing list