[llvm] 9e04895 - [SVE] Lower fixed length integer extend operations.
Paul Walker via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 03:56:36 PDT 2020
Author: Paul Walker
Date: 2020-08-13T11:54:53+01:00
New Revision: 9e04895258d0553982d92d2d67c4e6f4dd552beb
URL: https://github.com/llvm/llvm-project/commit/9e04895258d0553982d92d2d67c4e6f4dd552beb
DIFF: https://github.com/llvm/llvm-project/commit/9e04895258d0553982d92d2d67c4e6f4dd552beb.diff
LOG: [SVE] Lower fixed length integer extend operations.
Differential Revision: https://reviews.llvm.org/D85640
Added:
llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.h
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b9da2cf2bd38..6a2a1c7fcd1a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -128,6 +128,19 @@ static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock;
}
+// Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
+// predicate and end with a passthru value matching the result type.
+static bool isMergePassthruOpcode(unsigned Opc) {
+ switch (Opc) {
+ default:
+ return false;
+ case AArch64ISD::DUP_MERGE_PASSTHRU:
+ case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
+ case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
+ return true;
+ }
+}
+
AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
const AArch64Subtarget &STI)
: TargetLowering(TM), Subtarget(&STI) {
@@ -1082,6 +1095,7 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
// Lower fixed length vector operations to scalable equivalents.
setOperationAction(ISD::ADD, VT, Custom);
setOperationAction(ISD::AND, VT, Custom);
+ setOperationAction(ISD::ANY_EXTEND, VT, Custom);
setOperationAction(ISD::FADD, VT, Custom);
setOperationAction(ISD::FDIV, VT, Custom);
setOperationAction(ISD::FMA, VT, Custom);
@@ -1092,10 +1106,13 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
setOperationAction(ISD::LOAD, VT, Custom);
setOperationAction(ISD::MUL, VT, Custom);
setOperationAction(ISD::OR, VT, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
setOperationAction(ISD::STORE, VT, Custom);
setOperationAction(ISD::SUB, VT, Custom);
setOperationAction(ISD::TRUNCATE, VT, Custom);
setOperationAction(ISD::XOR, VT, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
}
void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
@@ -3666,6 +3683,13 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerDYNAMIC_STACKALLOC(Op, DAG);
case ISD::VSCALE:
return LowerVSCALE(Op, DAG);
+ case ISD::ANY_EXTEND:
+ case ISD::SIGN_EXTEND:
+ case ISD::ZERO_EXTEND:
+ return LowerFixedLengthVectorIntExtendToSVE(Op, DAG);
+ case ISD::SIGN_EXTEND_INREG:
+ return LowerToPredicatedOp(Op, DAG,
+ AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU);
case ISD::TRUNCATE:
return LowerTRUNCATE(Op, DAG);
case ISD::LOAD:
@@ -15283,6 +15307,42 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
Store->isTruncatingStore());
}
+SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
+ SDValue Op, SelectionDAG &DAG) const {
+ EVT VT = Op.getValueType();
+ assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+ SDLoc DL(Op);
+ SDValue Val = Op.getOperand(0);
+ EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
+ Val = convertToScalableVector(DAG, ContainerVT, Val);
+
+ bool Signed = Op.getOpcode() == ISD::SIGN_EXTEND;
+ unsigned ExtendOpc = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
+
+ // Repeatedly unpack Val until the result is of the desired element type.
+ switch (ContainerVT.getSimpleVT().SimpleTy) {
+ default:
+ llvm_unreachable("unimplemented container type");
+ case MVT::nxv16i8:
+ Val = DAG.getNode(ExtendOpc, DL, MVT::nxv8i16, Val);
+ if (VT.getVectorElementType() == MVT::i16)
+ break;
+ LLVM_FALLTHROUGH;
+ case MVT::nxv8i16:
+ Val = DAG.getNode(ExtendOpc, DL, MVT::nxv4i32, Val);
+ if (VT.getVectorElementType() == MVT::i32)
+ break;
+ LLVM_FALLTHROUGH;
+ case MVT::nxv4i32:
+ Val = DAG.getNode(ExtendOpc, DL, MVT::nxv2i64, Val);
+ assert(VT.getVectorElementType() == MVT::i64 && "Unexpected element type!");
+ break;
+ }
+
+ return convertFromScalableVector(DAG, VT, Val);
+}
+
SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
@@ -15321,6 +15381,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
// Convert vector operation 'Op' to an equivalent predicated operation whereby
// the original operation's type is used to construct a suitable predicate.
+// NOTE: The results for inactive lanes are undefined.
SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
SelectionDAG &DAG,
unsigned NewOp,
@@ -15340,11 +15401,21 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
continue;
}
+ if (const VTSDNode *VTNode = dyn_cast<VTSDNode>(V)) {
+ EVT VTArg = VTNode->getVT().getVectorElementType();
+ EVT NewVTArg = ContainerVT.changeVectorElementType(VTArg);
+ Operands.push_back(DAG.getValueType(NewVTArg));
+ continue;
+ }
+
assert(useSVEForFixedLengthVectorVT(V.getValueType(), OverrideNEON) &&
"Only fixed length vectors are supported!");
Operands.push_back(convertToScalableVector(DAG, ContainerVT, V));
}
+ if (isMergePassthruOpcode(NewOp))
+ Operands.push_back(DAG.getUNDEF(ContainerVT));
+
auto ScalableRes = DAG.getNode(NewOp, DL, ContainerVT, Operands);
return convertFromScalableVector(DAG, VT, ScalableRes);
}
@@ -15358,6 +15429,9 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
Operands.push_back(V);
}
+ if (isMergePassthruOpcode(NewOp))
+ Operands.push_back(DAG.getUNDEF(VT));
+
return DAG.getNode(NewOp, DL, VT, Operands);
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 78c29640a137..b7544b9439be 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -904,6 +904,8 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerSVEStructLoad(unsigned Intrinsic, ArrayRef<SDValue> LoadOps,
EVT VT, SelectionDAG &DAG, const SDLoc &DL) const;
+ SDValue LowerFixedLengthVectorIntExtendToSVE(SDValue Op,
+ SelectionDAG &DAG) const;
SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll
new file mode 100644
index 000000000000..cdeac8008133
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll
@@ -0,0 +1,747 @@
+; RUN: llc -aarch64-sve-vector-bits-min=128 -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_EQ_256
+; RUN: llc -aarch64-sve-vector-bits-min=384 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: z{0-9}
+
+;
+; sext i8 -> i16
+;
+
+define void @sext_v16i8_v16i16(<16 x i8> %a, <16 x i16>* %out) #0 {
+; CHECK-LABEL: sext_v16i8_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, z0.b
+; CHECK-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = sext <16 x i8> %a to <16 x i16>
+ store <16 x i16>%b, <16 x i16>* %out
+ ret void
+}
+
+; NOTE: Extra 'add' is to prevent the extend being combined with the load.
+define void @sext_v32i8_v32i16(<32 x i8>* %in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: sext_v32i8_v32i16:
+; VBITS_GE_512: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_512-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+ %a = load <32 x i8>, <32 x i8>* %in
+ %b = add <32 x i8> %a, %a
+ %c = sext <32 x i8> %b to <32 x i16>
+ store <32 x i16> %c, <32 x i16>* %out
+ ret void
+}
+
+define void @sext_v64i8_v64i16(<64 x i8>* %in, <64 x i16>* %out) #0 {
+; CHECK-LABEL: sext_v64i8_v64i16:
+; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_1024-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <64 x i8>, <64 x i8>* %in
+ %b = add <64 x i8> %a, %a
+ %c = sext <64 x i8> %b to <64 x i16>
+ store <64 x i16> %c, <64 x i16>* %out
+ ret void
+}
+
+define void @sext_v128i8_v128i16(<128 x i8>* %in, <128 x i16>* %out) #0 {
+; CHECK-LABEL: sext_v128i8_v128i16:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <128 x i8>, <128 x i8>* %in
+ %b = add <128 x i8> %a, %a
+ %c = sext <128 x i8> %b to <128 x i16>
+ store <128 x i16> %c, <128 x i16>* %out
+ ret void
+}
+
+;
+; sext i8 -> i32
+;
+
+define void @sext_v8i8_v8i32(<8 x i8> %a, <8 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v8i8_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, z0.b
+; CHECK-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; CHECK-NEXT: st1w { [[A_HALFS]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = sext <8 x i8> %a to <8 x i32>
+ store <8 x i32>%b, <8 x i32>* %out
+ ret void
+}
+
+define void @sext_v16i8_v16i32(<16 x i8> %a, <16 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v16i8_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256: ext v[[A_HI:[0-9]+]].16b, v0.16b, v0.16b, #8
+; VBITS_EQ_256-DAG: sunpklo [[A_HALFS_LO:z[0-9]+]].h, z0.b
+; VBITS_EQ_256-DAG: sunpklo [[A_HALFS_HI:z[0-9]+]].h, z[[A_HI]].b
+; VBITS_EQ_256-DAG: sunpklo [[A_WORDS_LO:z[0-9]+]].s, [[A_HALFS_LO]].h
+; VBITS_EQ_256-DAG: sunpklo [[A_WORDS_HI:z[0-9]+]].s, [[A_HALFS_HI]].h
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
+; VBITS_EQ_256-DAG: add x[[OUT_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: st1w { [[A_WORDS_LO]].s }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1w { [[A_WORDS_HI]].s }, [[PG]], [x[[OUT_HI]]]
+; VBITS_EQ_256-NEXT: ret
+ %b = sext <16 x i8> %a to <16 x i32>
+ store <16 x i32> %b, <16 x i32>* %out
+ ret void
+}
+
+define void @sext_v32i8_v32i32(<32 x i8>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v32i8_v32i32:
+; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_1024-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <32 x i8>, <32 x i8>* %in
+ %b = add <32 x i8> %a, %a
+ %c = sext <32 x i8> %b to <32 x i32>
+ store <32 x i32> %c, <32 x i32>* %out
+ ret void
+}
+
+define void @sext_v64i8_v64i32(<64 x i8>* %in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v64i8_v64i32:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <64 x i8>, <64 x i8>* %in
+ %b = add <64 x i8> %a, %a
+ %c = sext <64 x i8> %b to <64 x i32>
+ store <64 x i32> %c, <64 x i32>* %out
+ ret void
+}
+
+;
+; sext i8 -> i64
+;
+
+; NOTE: v4i8 is an unpacked typed stored within a v4i16 container. The sign
+; extend is a two step process where the container is any_extend'd with the
+; result feeding an inreg sign extend.
+define void @sext_v4i8_v4i64(<4 x i8> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v4i8_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[ANYEXT_W:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[ANYEXT_D:z[0-9]+]].d, [[ANYEXT_W]].s
+; CHECK-NEXT: sxtb [[A_DWORDS:z[0-9]+]].d, [[PG]]/m, [[ANYEXT_D]].d
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = sext <4 x i8> %a to <4 x i64>
+ store <4 x i64>%b, <4 x i64>* %out
+ ret void
+}
+
+define void @sext_v8i8_v8i64(<8 x i8> %a, <8 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v8i8_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+ %b = sext <8 x i8> %a to <8 x i64>
+ store <8 x i64>%b, <8 x i64>* %out
+ ret void
+}
+
+define void @sext_v16i8_v16i64(<16 x i8> %a, <16 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v16i8_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+ %b = sext <16 x i8> %a to <16 x i64>
+ store <16 x i64> %b, <16 x i64>* %out
+ ret void
+}
+
+define void @sext_v32i8_v32i64(<32 x i8>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v32i8_v32i64:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <32 x i8>, <32 x i8>* %in
+ %b = add <32 x i8> %a, %a
+ %c = sext <32 x i8> %b to <32 x i64>
+ store <32 x i64> %c, <32 x i64>* %out
+ ret void
+}
+
+;
+; sext i16 -> i32
+;
+
+define void @sext_v8i16_v8i32(<8 x i16> %a, <8 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v8i16_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = sext <8 x i16> %a to <8 x i32>
+ store <8 x i32>%b, <8 x i32>* %out
+ ret void
+}
+
+define void @sext_v16i16_v16i32(<16 x i16>* %in, <16 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v16i16_v16i32:
+; VBITS_GE_512: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+ %a = load <16 x i16>, <16 x i16>* %in
+ %b = add <16 x i16> %a, %a
+ %c = sext <16 x i16> %b to <16 x i32>
+ store <16 x i32> %c, <16 x i32>* %out
+ ret void
+}
+
+define void @sext_v32i16_v32i32(<32 x i16>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v32i16_v32i32:
+; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <32 x i16>, <32 x i16>* %in
+ %b = add <32 x i16> %a, %a
+ %c = sext <32 x i16> %b to <32 x i32>
+ store <32 x i32> %c, <32 x i32>* %out
+ ret void
+}
+
+define void @sext_v64i16_v64i32(<64 x i16>* %in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v64i16_v64i32:
+; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <64 x i16>, <64 x i16>* %in
+ %b = add <64 x i16> %a, %a
+ %c = sext <64 x i16> %b to <64 x i32>
+ store <64 x i32> %c, <64 x i32>* %out
+ ret void
+}
+
+;
+; sext i16 -> i64
+;
+
+define void @sext_v4i16_v4i64(<4 x i16> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v4i16_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = sext <4 x i16> %a to <4 x i64>
+ store <4 x i64>%b, <4 x i64>* %out
+ ret void
+}
+
+define void @sext_v8i16_v8i64(<8 x i16> %a, <8 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v8i16_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; VBITS_GE_512-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+ %b = sext <8 x i16> %a to <8 x i64>
+ store <8 x i64>%b, <8 x i64>* %out
+ ret void
+}
+
+define void @sext_v16i16_v16i64(<16 x i16>* %in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v16i16_v16i64:
+; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <16 x i16>, <16 x i16>* %in
+ %b = add <16 x i16> %a, %a
+ %c = sext <16 x i16> %b to <16 x i64>
+ store <16 x i64> %c, <16 x i64>* %out
+ ret void
+}
+
+define void @sext_v32i16_v32i64(<32 x i16>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v32i16_v32i64:
+; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <32 x i16>, <32 x i16>* %in
+ %b = add <32 x i16> %a, %a
+ %c = sext <32 x i16> %b to <32 x i64>
+ store <32 x i64> %c, <32 x i64>* %out
+ ret void
+}
+
+;
+; sext i32 -> i64
+;
+
+define void @sext_v4i32_v4i64(<4 x i32> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v4i32_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, z0.s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = sext <4 x i32> %a to <4 x i64>
+ store <4 x i64>%b, <4 x i64>* %out
+ ret void
+}
+
+define void @sext_v8i32_v8i64(<8 x i32>* %in, <8 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v8i32_v8i64:
+; VBITS_GE_512: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_512-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+ %a = load <8 x i32>, <8 x i32>* %in
+ %b = add <8 x i32> %a, %a
+ %c = sext <8 x i32> %b to <8 x i64>
+ store <8 x i64> %c, <8 x i64>* %out
+ ret void
+}
+
+define void @sext_v16i32_v16i64(<16 x i32>* %in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v16i32_v16i64:
+; VBITS_GE_1024: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_1024-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <16 x i32>, <16 x i32>* %in
+ %b = add <16 x i32> %a, %a
+ %c = sext <16 x i32> %b to <16 x i64>
+ store <16 x i64> %c, <16 x i64>* %out
+ ret void
+}
+
+define void @sext_v32i32_v32i64(<32 x i32>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v32i32_v32i64:
+; VBITS_GE_2048: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <32 x i32>, <32 x i32>* %in
+ %b = add <32 x i32> %a, %a
+ %c = sext <32 x i32> %b to <32 x i64>
+ store <32 x i64> %c, <32 x i64>* %out
+ ret void
+}
+
+;
+; zext i8 -> i16
+;
+
+define void @zext_v16i8_v16i16(<16 x i8> %a, <16 x i16>* %out) #0 {
+; CHECK-LABEL: zext_v16i8_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, z0.b
+; CHECK-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = zext <16 x i8> %a to <16 x i16>
+ store <16 x i16>%b, <16 x i16>* %out
+ ret void
+}
+
+; NOTE: Extra 'add' is to prevent the extend being combined with the load.
+define void @zext_v32i8_v32i16(<32 x i8>* %in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: zext_v32i8_v32i16:
+; VBITS_GE_512: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_512-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+ %a = load <32 x i8>, <32 x i8>* %in
+ %b = add <32 x i8> %a, %a
+ %c = zext <32 x i8> %b to <32 x i16>
+ store <32 x i16> %c, <32 x i16>* %out
+ ret void
+}
+
+define void @zext_v64i8_v64i16(<64 x i8>* %in, <64 x i16>* %out) #0 {
+; CHECK-LABEL: zext_v64i8_v64i16:
+; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_1024-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <64 x i8>, <64 x i8>* %in
+ %b = add <64 x i8> %a, %a
+ %c = zext <64 x i8> %b to <64 x i16>
+ store <64 x i16> %c, <64 x i16>* %out
+ ret void
+}
+
+define void @zext_v128i8_v128i16(<128 x i8>* %in, <128 x i16>* %out) #0 {
+; CHECK-LABEL: zext_v128i8_v128i16:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <128 x i8>, <128 x i8>* %in
+ %b = add <128 x i8> %a, %a
+ %c = zext <128 x i8> %b to <128 x i16>
+ store <128 x i16> %c, <128 x i16>* %out
+ ret void
+}
+
+;
+; zext i8 -> i32
+;
+
+define void @zext_v8i8_v8i32(<8 x i8> %a, <8 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v8i8_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, z0.b
+; CHECK-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; CHECK-NEXT: st1w { [[A_HALFS]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = zext <8 x i8> %a to <8 x i32>
+ store <8 x i32>%b, <8 x i32>* %out
+ ret void
+}
+
+define void @zext_v16i8_v16i32(<16 x i8> %a, <16 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v16i8_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256: ext v[[A_HI:[0-9]+]].16b, v0.16b, v0.16b, #8
+; VBITS_EQ_256-DAG: uunpklo [[A_HALFS_LO:z[0-9]+]].h, z0.b
+; VBITS_EQ_256-DAG: uunpklo [[A_HALFS_HI:z[0-9]+]].h, z[[A_HI]].b
+; VBITS_EQ_256-DAG: uunpklo [[A_WORDS_LO:z[0-9]+]].s, [[A_HALFS_LO]].h
+; VBITS_EQ_256-DAG: uunpklo [[A_WORDS_HI:z[0-9]+]].s, [[A_HALFS_HI]].h
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
+; VBITS_EQ_256-DAG: add x[[OUT_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: st1w { [[A_WORDS_LO]].s }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1w { [[A_WORDS_HI]].s }, [[PG]], [x[[OUT_HI]]]
+; VBITS_EQ_256-NEXT: ret
+ %b = zext <16 x i8> %a to <16 x i32>
+ store <16 x i32> %b, <16 x i32>* %out
+ ret void
+}
+
+define void @zext_v32i8_v32i32(<32 x i8>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v32i8_v32i32:
+; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_1024-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <32 x i8>, <32 x i8>* %in
+ %b = add <32 x i8> %a, %a
+ %c = zext <32 x i8> %b to <32 x i32>
+ store <32 x i32> %c, <32 x i32>* %out
+ ret void
+}
+
+define void @zext_v64i8_v64i32(<64 x i8>* %in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v64i8_v64i32:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <64 x i8>, <64 x i8>* %in
+ %b = add <64 x i8> %a, %a
+ %c = zext <64 x i8> %b to <64 x i32>
+ store <64 x i32> %c, <64 x i32>* %out
+ ret void
+}
+
+;
+; zext i8 -> i64
+;
+
+; NOTE: v4i8 is an unpacked typed stored within a v4i16 container. The zero
+; extend is a two step process where the container is zero_extend_inreg'd with
+; the result feeding a normal zero extend from halfs to doublewords.
+define void @zext_v4i8_v4i64(<4 x i8> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v4i8_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = zext <4 x i8> %a to <4 x i64>
+ store <4 x i64>%b, <4 x i64>* %out
+ ret void
+}
+
+define void @zext_v8i8_v8i64(<8 x i8> %a, <8 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v8i8_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+ %b = zext <8 x i8> %a to <8 x i64>
+ store <8 x i64>%b, <8 x i64>* %out
+ ret void
+}
+
+define void @zext_v16i8_v16i64(<16 x i8> %a, <16 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v16i8_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+ %b = zext <16 x i8> %a to <16 x i64>
+ store <16 x i64> %b, <16 x i64>* %out
+ ret void
+}
+
+define void @zext_v32i8_v32i64(<32 x i8>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v32i8_v32i64:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <32 x i8>, <32 x i8>* %in
+ %b = add <32 x i8> %a, %a
+ %c = zext <32 x i8> %b to <32 x i64>
+ store <32 x i64> %c, <32 x i64>* %out
+ ret void
+}
+
+;
+; zext i16 -> i32
+;
+
+define void @zext_v8i16_v8i32(<8 x i16> %a, <8 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v8i16_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = zext <8 x i16> %a to <8 x i32>
+ store <8 x i32>%b, <8 x i32>* %out
+ ret void
+}
+
+define void @zext_v16i16_v16i32(<16 x i16>* %in, <16 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v16i16_v16i32:
+; VBITS_GE_512: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+ %a = load <16 x i16>, <16 x i16>* %in
+ %b = add <16 x i16> %a, %a
+ %c = zext <16 x i16> %b to <16 x i32>
+ store <16 x i32> %c, <16 x i32>* %out
+ ret void
+}
+
+define void @zext_v32i16_v32i32(<32 x i16>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v32i16_v32i32:
+; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <32 x i16>, <32 x i16>* %in
+ %b = add <32 x i16> %a, %a
+ %c = zext <32 x i16> %b to <32 x i32>
+ store <32 x i32> %c, <32 x i32>* %out
+ ret void
+}
+
+define void @zext_v64i16_v64i32(<64 x i16>* %in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v64i16_v64i32:
+; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <64 x i16>, <64 x i16>* %in
+ %b = add <64 x i16> %a, %a
+ %c = zext <64 x i16> %b to <64 x i32>
+ store <64 x i32> %c, <64 x i32>* %out
+ ret void
+}
+
+;
+; zext i16 -> i64
+;
+
+define void @zext_v4i16_v4i64(<4 x i16> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v4i16_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = zext <4 x i16> %a to <4 x i64>
+ store <4 x i64>%b, <4 x i64>* %out
+ ret void
+}
+
+define void @zext_v8i16_v8i64(<8 x i16> %a, <8 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v8i16_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; VBITS_GE_512-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+ %b = zext <8 x i16> %a to <8 x i64>
+ store <8 x i64>%b, <8 x i64>* %out
+ ret void
+}
+
+define void @zext_v16i16_v16i64(<16 x i16>* %in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v16i16_v16i64:
+; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <16 x i16>, <16 x i16>* %in
+ %b = add <16 x i16> %a, %a
+ %c = zext <16 x i16> %b to <16 x i64>
+ store <16 x i64> %c, <16 x i64>* %out
+ ret void
+}
+
+define void @zext_v32i16_v32i64(<32 x i16>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v32i16_v32i64:
+; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <32 x i16>, <32 x i16>* %in
+ %b = add <32 x i16> %a, %a
+ %c = zext <32 x i16> %b to <32 x i64>
+ store <32 x i64> %c, <32 x i64>* %out
+ ret void
+}
+
+;
+; zext i32 -> i64
+;
+
+define void @zext_v4i32_v4i64(<4 x i32> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v4i32_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, z0.s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+ %b = zext <4 x i32> %a to <4 x i64>
+ store <4 x i64>%b, <4 x i64>* %out
+ ret void
+}
+
+define void @zext_v8i32_v8i64(<8 x i32>* %in, <8 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v8i32_v8i64:
+; VBITS_GE_512: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_512-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+ %a = load <8 x i32>, <8 x i32>* %in
+ %b = add <8 x i32> %a, %a
+ %c = zext <8 x i32> %b to <8 x i64>
+ store <8 x i64> %c, <8 x i64>* %out
+ ret void
+}
+
+define void @zext_v16i32_v16i64(<16 x i32>* %in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v16i32_v16i64:
+; VBITS_GE_1024: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_1024-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+ %a = load <16 x i32>, <16 x i32>* %in
+ %b = add <16 x i32> %a, %a
+ %c = zext <16 x i32> %b to <16 x i64>
+ store <16 x i64> %c, <16 x i64>* %out
+ ret void
+}
+
+define void @zext_v32i32_v32i64(<32 x i32>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v32i32_v32i64:
+; VBITS_GE_2048: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+ %a = load <32 x i32>, <32 x i32>* %in
+ %b = add <32 x i32> %a, %a
+ %c = zext <32 x i32> %b to <32 x i64>
+ store <32 x i64> %c, <32 x i64>* %out
+ ret void
+}
+
+attributes #0 = { nounwind "target-features"="+sve" }
More information about the llvm-commits
mailing list