[llvm] [AArch64] Use ADDP tree for v16i8 to i16 bitmask extraction (PR #192974)
Conor Kotwasinski via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 23:52:40 PDT 2026
https://github.com/conorKotwasinski updated https://github.com/llvm/llvm-project/pull/192974
>From 11cb3f8359ee3fda1dd535f87d553e753109cade Mon Sep 17 00:00:00 2001
From: Conor Kotwasinski <conorkotwasinski2024 at u.northwestern.edu>
Date: Thu, 30 Apr 2026 02:39:24 -0400
Subject: [PATCH] [AArch64] Use ADDP tree for v16i8 to i16 bitmask extraction
Before:
ext v1.16b, v0.16b, v0.16b, #8
zip1 v0.16b, v0.16b, v1.16b
addv h0, v0.8h
fmov w0, s0
After:
addp v0.16b, v0.16b, v0.16b
addp v0.16b, v0.16b, v0.16b
addp v0.16b, v0.16b, v0.16b
umov w0, v0.h[0]
The existing lowering in vectorToScalarBitmask for v16i8 used an EXT+ZIP1+ADDV
sequence to pack the per-lane bits into an i16. The horizontal ADDV is
expensive on some microarchitectures and forces an early SIMD-to-GPR domain
crossing.
Replace it with a three-level pairwise-add (ADDP) tree. After the AND with
the powers-of-two constant, each byte lane holds at most one set bit within
its 8-byte group, so pairwise addition cannot carry and is equivalent to OR
at every level. Three ADDPs therefore losslessly pack the 16 per-lane bits
into the low i16 of the result vector, from which we extract the bitmask
with a single UMOV.
This keeps all computation in the vector register file until the final
extract and avoids the horizontal reduction.
Partially addresses #192749.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 222 ++------------
llvm/test/CodeGen/AArch64/alias_mask.ll | 283 ++++++++----------
.../test/CodeGen/AArch64/dag-combine-setcc.ll | 32 +-
.../fixed_masked_deinterleaved_loads.ll | 24 +-
.../fixed_masked_interleaved_stores.ll | 24 +-
.../sve-fixed-length-masked-expandloads.ll | 98 +++---
.../CodeGen/AArch64/sve-mask-partition.ll | 12 +-
.../AArch64/vec-combine-compare-to-bitmask.ll | 22 +-
.../vec-combine-compare-truncate-store.ll | 6 +-
9 files changed, 269 insertions(+), 454 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b23bbd7234177..ae7080d5988f7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1153,9 +1153,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setTargetDAGCombine({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
ISD::FP_TO_UINT_SAT, ISD::FADD});
- // Try and combine setcc/select_cc with csel and bool-vector bitcasts.
+ // Try and combine setcc with csel
setTargetDAGCombine(ISD::SETCC);
- setTargetDAGCombine(ISD::SELECT_CC);
setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
@@ -5465,7 +5464,6 @@ AArch64TargetLowering::LowerLOOP_DEPENDENCE_MASK(SDValue Op,
unsigned LaneOffset = Op.getConstantOperandVal(3);
unsigned NumElements = VT.getVectorMinNumElements();
uint64_t EltSizeInBytes = Op.getConstantOperandVal(2);
- EVT AddrTy = Op->getOperand(0).getValueType();
// Lane offsets and other element sizes are not supported by whilewr/rw.
if (LaneOffset != 0 || !is_contained({1u, 2u, 4u, 8u}, EltSizeInBytes))
@@ -5475,19 +5473,9 @@ AArch64TargetLowering::LowerLOOP_DEPENDENCE_MASK(SDValue Op,
EVT PredVT =
getPackedSVEVectorVT(EltVT).changeElementType(*DAG.getContext(), MVT::i1);
- if (PredVT == VT) {
- // Legal whilewr/rw (lowered by tablegen matcher).
- if (AddrTy == MVT::i64)
- return Op;
-
- // Almost legal whilewr/rw (addresses must be promoted to i64).
- assert(AddrTy == MVT::i32 && "Only expected i32 to be legal!");
- return DAG.getNode(
- Op.getOpcode(), DL, VT,
- DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Op->getOperand(0)),
- DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Op->getOperand(1)),
- DAG.getConstant(EltSizeInBytes, DL, MVT::i64), Op->getOperand(3));
- }
+ // Legal whilewr/rw (lowered by tablegen matcher).
+ if (PredVT == VT)
+ return Op;
// Expand if this mask needs splitting (this will produce a whilelo).
if (NumElements > PredVT.getVectorMinNumElements())
@@ -25736,68 +25724,6 @@ static EVT tryGetOriginalBoolVectorType(SDValue Op, int Depth = 0) {
return BaseVT;
}
-static bool getBoolVectorBitcastCompare(SDValue Vec, SDValue RHS,
- const SDLoc &DL, SelectionDAG &DAG,
- SDValue &CompareLHS,
- SDValue &CompareRHS) {
- if (DAG.getDataLayout().isBigEndian())
- return false;
-
- EVT VecVT = Vec.getValueType();
- assert(VecVT.isFixedLengthVector() &&
- VecVT.getVectorElementType() == MVT::i1 &&
- "Expected a fixed-length bool vector");
-
- unsigned NumElts = VecVT.getVectorNumElements();
- if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
- return false;
-
- auto getCanonicalCompareVecVT = [&]() {
- unsigned BitsPerElement = std::max(64 / NumElts, 8u);
- return MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), NumElts);
- };
-
- EVT CompareVecVT = tryGetOriginalBoolVectorType(Vec);
- if (!CompareVecVT.isSimple() || CompareVecVT.getSizeInBits() > 128) {
- CompareVecVT = getCanonicalCompareVecVT();
- }
- CompareVecVT = CompareVecVT.changeVectorElementTypeToInteger();
-
- if (CompareVecVT.getSizeInBits() > 128)
- return false;
-
- SDValue CompareBits = DAG.getSExtOrTrunc(Vec, DL, CompareVecVT);
- unsigned CompareBitsSize = CompareBits.getValueSizeInBits();
-
- // Use a canonical 64/128-bit vector representation before bitcasting to a
- // scalar view. Some legal original vector types are smaller than 64-bit,
- // which would make the direct scalar bitcasts below invalid.
- if (CompareBitsSize != 64 && CompareBitsSize != 128) {
- CompareVecVT = getCanonicalCompareVecVT();
- CompareBits = DAG.getSExtOrTrunc(Vec, DL, CompareVecVT);
- CompareBitsSize = CompareBits.getValueSizeInBits();
- }
-
- if (CompareBitsSize != 64 && CompareBitsSize != 128)
- return false;
-
- bool IsNull = isNullConstant(RHS);
- if (CompareBitsSize == 64) {
- CompareLHS = DAG.getBitcast(MVT::i64, CompareBits);
- CompareRHS = IsNull ? DAG.getConstant(0, DL, MVT::i64)
- : DAG.getAllOnesConstant(DL, MVT::i64);
- } else {
- SDValue PairwiseBits = DAG.getBitcast(MVT::v2i64, CompareBits);
- SDValue Lo = DAG.getExtractVectorElt(DL, MVT::i64, PairwiseBits, 0);
- SDValue Hi = DAG.getExtractVectorElt(DL, MVT::i64, PairwiseBits, 1);
- CompareLHS = DAG.getNode(ISD::ADD, DL, MVT::i64, Lo, Hi);
- CompareRHS = IsNull ? DAG.getConstant(0, DL, MVT::i64)
- : DAG.getSignedConstant(-2, DL, MVT::i64);
- }
-
- return true;
-}
-
// When converting a <N x iX> vector to <N x i1> to store or use as a scalar
// iN, we can use a trick that extracts the i^th bit from the i^th element and
// then performs a vector add to get a scalar bitmask. This requires that each
@@ -25840,9 +25766,12 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
SmallVector<SDValue, 16> MaskConstants;
if (DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable() &&
VecVT == MVT::v16i8) {
- // v16i8 is a special case, as we have 16 entries but only 8 positional bits
- // per entry. We split it into two halves, apply the mask, zip the halves to
- // create 8x 16-bit values, and the perform the vector reduce.
+ // v16i8 is a special case: we have 16 entries but only 8 positional bits
+ // per i16 of bitmask. ANDing with a mask whose two 8-byte halves both
+ // hold the powers-of-two 1,2,...,128 leaves at most one bit set per byte
+ // lane within each 8-byte group, so pairwise addition equals OR and
+ // cannot carry. Three levels of ADDP then losslessly pack the 16
+ // per-lane bits into the low i16 of the result vector.
for (unsigned Half = 0; Half < 2; ++Half) {
for (unsigned I = 0; I < 8; ++I) {
// On big-endian targets, the lane order in sub-byte vector elements
@@ -25855,13 +25784,13 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
SDValue RepresentativeBits =
DAG.getNode(ISD::AND, DL, VecVT, ComparisonResult, Mask);
- SDValue UpperRepresentativeBits =
- DAG.getNode(AArch64ISD::EXT, DL, VecVT, RepresentativeBits,
- RepresentativeBits, DAG.getConstant(8, DL, MVT::i32));
- SDValue Zipped = DAG.getNode(AArch64ISD::ZIP1, DL, VecVT,
- RepresentativeBits, UpperRepresentativeBits);
- Zipped = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Zipped);
- return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i16, Zipped);
+ SDValue V = RepresentativeBits;
+ V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V);
+ V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V);
+ V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V);
+ V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V);
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, V,
+ DAG.getConstant(0, DL, MVT::i64));
}
// All other vector sizes.
@@ -26424,6 +26353,17 @@ static bool findMoreOptimalIndexType(const MaskedGatherScatterSDNode *N,
if (Index.getOpcode() == ISD::STEP_VECTOR) {
Stride = cast<ConstantSDNode>(Index.getOperand(0))->getSExtValue();
}
+ // Match:
+ // Index = step(const) << shift(const)
+ else if (Index.getOpcode() == ISD::SHL &&
+ Index.getOperand(0).getOpcode() == ISD::STEP_VECTOR) {
+ SDValue RHS = Index.getOperand(1);
+ if (auto *Shift =
+ dyn_cast_or_null<ConstantSDNode>(DAG.getSplatValue(RHS))) {
+ int64_t Step = (int64_t)Index.getOperand(0).getConstantOperandVal(1);
+ Stride = Step << Shift->getZExtValue();
+ }
+ }
// Return early because no supported pattern is found.
if (Stride == 0)
@@ -27483,9 +27423,10 @@ static SDValue performSETCCCombine(SDNode *N,
}
}
- // When a bool vector bitcast is only compared against zero or all ones, it
- // is enough to test a widened scalar view of the comparison bits. This
- // avoids materializing the packed bitmask via vectorToScalarBitmask().
+ // setcc (iN (bitcast (vNi1 X))), 0, (eq|ne)
+ // ==> setcc (iN (zext (i1 (vecreduce_or (vNi1 X))))), 0, (eq|ne)
+ // setcc (iN (bitcast (vNi1 X))), -1, (eq|ne)
+ // ==> setcc (iN (sext (i1 (vecreduce_and (vNi1 X))))), -1, (eq|ne)
if (DCI.isBeforeLegalize() && VT.isScalarInteger() &&
(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
(isNullConstant(RHS) || isAllOnesConstant(RHS)) &&
@@ -27494,11 +27435,6 @@ static SDValue performSETCCCombine(SDNode *N,
EVT FromVT = LHS->getOperand(0).getValueType();
if (FromVT.isFixedLengthVector() &&
FromVT.getVectorElementType() == MVT::i1) {
- SDValue CompareLHS, CompareRHS;
- if (getBoolVectorBitcastCompare(LHS.getOperand(0), RHS, DL, DAG,
- CompareLHS, CompareRHS))
- return DAG.getSetCC(DL, VT, CompareLHS, CompareRHS, Cond);
-
bool IsNull = isNullConstant(RHS);
LHS = DAG.getNode(IsNull ? ISD::VECREDUCE_OR : ISD::VECREDUCE_AND,
DL, MVT::i1, LHS->getOperand(0));
@@ -27525,43 +27461,6 @@ static SDValue performSETCCCombine(SDNode *N,
return SDValue();
}
-static SDValue performSELECT_CCCombine(SDNode *N,
- TargetLowering::DAGCombinerInfo &DCI,
- SelectionDAG &DAG) {
- assert(N->getOpcode() == ISD::SELECT_CC && "Unexpected opcode!");
-
- if (!DCI.isBeforeLegalize())
- return SDValue();
-
- SDValue LHS = N->getOperand(0);
- SDValue RHS = N->getOperand(1);
- ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(4))->get();
- if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
- return SDValue();
-
- if ((isNullConstant(LHS) || isAllOnesConstant(LHS)) &&
- RHS.getOpcode() == ISD::BITCAST)
- std::swap(LHS, RHS);
-
- if (!(isNullConstant(RHS) || isAllOnesConstant(RHS)) ||
- LHS.getOpcode() != ISD::BITCAST)
- return SDValue();
-
- EVT FromVT = LHS.getOperand(0).getValueType();
- if (!FromVT.isFixedLengthVector() || FromVT.getVectorElementType() != MVT::i1)
- return SDValue();
-
- SDLoc DL(N);
- SDValue CompareLHS, CompareRHS;
- if (!getBoolVectorBitcastCompare(LHS.getOperand(0), RHS, DL, DAG, CompareLHS,
- CompareRHS))
- return SDValue();
-
- return DAG.getNode(ISD::SELECT_CC, DL, N->getValueType(0), CompareLHS,
- CompareRHS, N->getOperand(2), N->getOperand(3),
- N->getOperand(4));
-}
-
// Replace a flag-setting operator (eg ANDS) with the generic version
// (eg AND) if the flag is unused.
static SDValue performFlagSettingCombine(SDNode *N,
@@ -27866,58 +27765,6 @@ static SDValue trySwapVSelectOperands(SDNode *N, SelectionDAG &DAG) {
{InverseSetCC, SelectB, SelectA});
}
-static SDValue performVselectPowCombine(SDNode *N,
- TargetLowering::DAGCombinerInfo &DCI) {
- assert(N->getOpcode() == ISD::VSELECT && "Expected VSELECT opcode");
- SDValue Cond = N->getOperand(0);
- SDValue TrueVal = N->getOperand(1);
- SDValue FalseVal = N->getOperand(2);
- bool TrueValIsPow = TrueVal.getOpcode() == ISD::FPOW;
- bool FalseValIsPow = FalseVal.getOpcode() == ISD::FPOW;
-
- // If both inputs are pow we could equally remove the select and simply
- // select between pow inputs instead.
- if (TrueValIsPow == FalseValIsPow)
- return SDValue();
-
- if ((TrueValIsPow && !TrueVal.hasOneUse()) ||
- (FalseValIsPow && !FalseVal.hasOneUse()))
- return SDValue();
-
- EVT VT = N->getValueType(0);
- RTLIB::Libcall LC = RTLIB::getPOW(VT);
- SelectionDAG &DAG = DCI.DAG;
- auto &TLI = DAG.getTargetLoweringInfo();
- bool HasLibCall =
- TLI.getLibcallLoweringInfo().getLibcallImpl(LC) != RTLIB::Unsupported;
- if (!HasLibCall)
- return SDValue();
-
- SDValue OldPow = TrueValIsPow ? TrueVal : FalseVal;
- SDValue OldPowArg0 = OldPow->getOperand(0);
-
- // Bail out if argument 0 is already a select, in order to avoid an infinite
- // combine loop.
- if (OldPowArg0.getOpcode() == ISD::VSELECT)
- return SDValue();
-
- // For a given call pow(x, y) when x=1.0 it is guaranteed to return 1.0 for
- // any value of y.
- SDLoc DL(N);
- SDValue SplatOne = DAG.getConstantFP(1.0, DL, VT);
- SDValue NewPowArg0;
- if (TrueValIsPow)
- NewPowArg0 = DAG.getNode(ISD::VSELECT, DL, VT, Cond, OldPowArg0, SplatOne);
- else
- NewPowArg0 = DAG.getNode(ISD::VSELECT, DL, VT, Cond, SplatOne, OldPowArg0);
- SDValue NewPow = DAG.getNode(ISD::FPOW, DL, VT, NewPowArg0,
- OldPow->getOperand(1), OldPow->getFlags());
-
- if (TrueValIsPow)
- return DAG.getNode(ISD::VSELECT, DL, VT, Cond, NewPow, FalseVal);
- return DAG.getNode(ISD::VSELECT, DL, VT, Cond, TrueVal, NewPow);
-}
-
// vselect (v1i1 setcc) ->
// vselect (v1iXX setcc) (XX is the size of the compared operand type)
// FIXME: Currently the type legalizer can't handle VSELECT having v1i1 as
@@ -28005,9 +27852,6 @@ static SDValue performVSelectCombine(SDNode *N,
}
}
- if (SDValue R = performVselectPowCombine(N, DCI))
- return R;
-
EVT CmpVT = N0.getOperand(0).getValueType();
if (N0.getOpcode() != ISD::SETCC ||
CCVT.getVectorElementCount() != ElementCount::getFixed(1) ||
@@ -29443,8 +29287,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performInsertSubvectorCombine(N, DCI, DAG);
case ISD::SELECT:
return performSelectCombine(N, DCI);
- case ISD::SELECT_CC:
- return performSELECT_CCCombine(N, DCI, DAG);
case ISD::VSELECT:
return performVSelectCombine(N, DCI, Subtarget);
case ISD::SETCC:
diff --git a/llvm/test/CodeGen/AArch64/alias_mask.ll b/llvm/test/CodeGen/AArch64/alias_mask.ll
index d3e6513c83cd6..3275afbbd33f9 100644
--- a/llvm/test/CodeGen/AArch64/alias_mask.ll
+++ b/llvm/test/CodeGen/AArch64/alias_mask.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64 -mattr=+sve2 %s -o - | FileCheck %s
-define <16 x i1> @whilewr_8(i64 %a, i64 %b) {
+define <16 x i1> @whilewr_8(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.b, x0, x1
@@ -9,11 +9,11 @@ define <16 x i1> @whilewr_8(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 1)
+ %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 1)
ret <16 x i1> %0
}
-define <8 x i1> @whilewr_16(i64 %a, i64 %b) {
+define <8 x i1> @whilewr_16(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.h, x0, x1
@@ -21,11 +21,11 @@ define <8 x i1> @whilewr_16(i64 %a, i64 %b) {
; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: ret
entry:
- %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 2)
+ %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 2)
ret <8 x i1> %0
}
-define <4 x i1> @whilewr_32(i64 %a, i64 %b) {
+define <4 x i1> @whilewr_32(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.s, x0, x1
@@ -33,11 +33,11 @@ define <4 x i1> @whilewr_32(i64 %a, i64 %b) {
; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
entry:
- %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr %a, ptr %b, i64 4)
ret <4 x i1> %0
}
-define <2 x i1> @whilewr_64(i64 %a, i64 %b) {
+define <2 x i1> @whilewr_64(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.d, x0, x1
@@ -45,11 +45,11 @@ define <2 x i1> @whilewr_64(i64 %a, i64 %b) {
; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: ret
entry:
- %0 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1.i64(i64 %a, i64 %b, i64 8)
+ %0 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1(ptr %a, ptr %b, i64 8)
ret <2 x i1> %0
}
-define <16 x i1> @whilerw_8(i64 %a, i64 %b) {
+define <16 x i1> @whilerw_8(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilerw p0.b, x0, x1
@@ -57,11 +57,11 @@ define <16 x i1> @whilerw_8(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1.i64(i64 %a, i64 %b, i64 1)
+ %0 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1(ptr %a, ptr %b, i64 1)
ret <16 x i1> %0
}
-define <8 x i1> @whilerw_16(i64 %a, i64 %b) {
+define <8 x i1> @whilerw_16(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_16:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilerw p0.h, x0, x1
@@ -69,11 +69,11 @@ define <8 x i1> @whilerw_16(i64 %a, i64 %b) {
; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: ret
entry:
- %0 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1.i64(i64 %a, i64 %b, i64 2)
+ %0 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1(ptr %a, ptr %b, i64 2)
ret <8 x i1> %0
}
-define <4 x i1> @whilerw_32(i64 %a, i64 %b) {
+define <4 x i1> @whilerw_32(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_32:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilerw p0.s, x0, x1
@@ -81,11 +81,11 @@ define <4 x i1> @whilerw_32(i64 %a, i64 %b) {
; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
entry:
- %0 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1(ptr %a, ptr %b, i64 4)
ret <4 x i1> %0
}
-define <2 x i1> @whilerw_64(i64 %a, i64 %b) {
+define <2 x i1> @whilerw_64(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_64:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilerw p0.d, x0, x1
@@ -93,11 +93,11 @@ define <2 x i1> @whilerw_64(i64 %a, i64 %b) {
; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: ret
entry:
- %0 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1.i64(i64 %a, i64 %b, i64 8)
+ %0 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1(ptr %a, ptr %b, i64 8)
ret <2 x i1> %0
}
-define <32 x i1> @whilewr_8_split(i64 %a, i64 %b) {
+define <32 x i1> @whilewr_8_split(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8_split:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sub x9, x1, x0
@@ -112,66 +112,67 @@ define <32 x i1> @whilewr_8_split(i64 %a, i64 %b) {
; CHECK-NEXT: mov z1.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: zip1 v1.16b, v1.16b, v3.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT: str h0, [x8]
; CHECK-NEXT: str h1, [x8, #2]
; CHECK-NEXT: ret
entry:
- %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1.i64(i64 %a, i64 %b, i64 1)
+ %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1(ptr %a, ptr %b, i64 1)
ret <32 x i1> %0
}
-define <64 x i1> @whilewr_8_split2(i64 %a, i64 %b) {
+define <64 x i1> @whilewr_8_split2(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8_split2:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: sub x9, x1, x0
-; CHECK-NEXT: mov w10, #48 // =0x30
-; CHECK-NEXT: mov w11, #16 // =0x10
-; CHECK-NEXT: cmp x9, #1
-; CHECK-NEXT: mov w12, #32 // =0x20
-; CHECK-NEXT: csinv x9, x9, xzr, ge
-; CHECK-NEXT: whilewr p0.b, x0, x1
-; CHECK-NEXT: whilelo p1.b, x10, x9
-; CHECK-NEXT: adrp x10, .LCPI9_0
+; CHECK-NEXT: sub x10, x1, x0
+; CHECK-NEXT: mov w9, #32 // =0x20
+; CHECK-NEXT: mov w11, #48 // =0x30
+; CHECK-NEXT: cmp x10, #1
+; CHECK-NEXT: csinv x10, x10, xzr, ge
+; CHECK-NEXT: whilelo p0.b, x9, x10
+; CHECK-NEXT: mov w9, #16 // =0x10
+; CHECK-NEXT: whilelo p1.b, x11, x10
+; CHECK-NEXT: adrp x11, .LCPI9_0
; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: whilelo p0.b, x12, x9
-; CHECK-NEXT: ldr q1, [x10, :lo12:.LCPI9_0]
-; CHECK-NEXT: mov z2.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: whilelo p1.b, x11, x9
+; CHECK-NEXT: ldr q2, [x11, :lo12:.LCPI9_0]
+; CHECK-NEXT: whilelo p0.b, x9, x10
+; CHECK-NEXT: mov z1.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: whilewr p1.b, x0, x1
; CHECK-NEXT: mov z3.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: mov z4.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: and v2.16b, v2.16b, v1.16b
-; CHECK-NEXT: and v3.16b, v3.16b, v1.16b
-; CHECK-NEXT: and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v5.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: ext v6.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT: ext v7.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v4.16b
-; CHECK-NEXT: zip1 v2.16b, v2.16b, v5.16b
-; CHECK-NEXT: zip1 v3.16b, v3.16b, v6.16b
-; CHECK-NEXT: zip1 v1.16b, v1.16b, v7.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: addv h2, v2.8h
-; CHECK-NEXT: addv h3, v3.8h
-; CHECK-NEXT: addv h1, v1.8h
-; CHECK-NEXT: str h0, [x8]
-; CHECK-NEXT: str h2, [x8, #6]
-; CHECK-NEXT: str h3, [x8, #4]
-; CHECK-NEXT: str h1, [x8, #2]
-; CHECK-NEXT: ret
-entry:
- %0 = call <64 x i1> @llvm.loop.dependence.war.mask.v64i1.i64(i64 %a, i64 %b, i64 1)
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: and v3.16b, v3.16b, v2.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-NEXT: zip1 v1.4h, v0.4h, v1.4h
+; CHECK-NEXT: ext v0.8b, v0.8b, v1.8b, #4
+; CHECK-NEXT: zip1 v1.4h, v2.4h, v3.4h
+; CHECK-NEXT: mov v1.s[1], v0.s[1]
+; CHECK-NEXT: str d1, [x8]
+; CHECK-NEXT: ret
+entry:
+ %0 = call <64 x i1> @llvm.loop.dependence.war.mask.v64i1(ptr %a, ptr %b, i64 1)
ret <64 x i1> %0
}
-define <16 x i1> @whilewr_16_expand(i64 %a, i64 %b) {
+define <16 x i1> @whilewr_16_expand(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16_expand:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sub x8, x1, x0
@@ -184,11 +185,11 @@ define <16 x i1> @whilewr_16_expand(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 2)
+ %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 2)
ret <16 x i1> %0
}
-define <32 x i1> @whilewr_16_expand2(i64 %a, i64 %b) {
+define <32 x i1> @whilewr_16_expand2(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16_expand2:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sub x9, x1, x0
@@ -205,21 +206,21 @@ define <32 x i1> @whilewr_16_expand2(i64 %a, i64 %b) {
; CHECK-NEXT: mov z2.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: zip1 v1.16b, v1.16b, v3.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT: str h0, [x8, #2]
; CHECK-NEXT: str h1, [x8]
; CHECK-NEXT: ret
entry:
- %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1.i64(i64 %a, i64 %b, i64 2)
+ %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1(ptr %a, ptr %b, i64 2)
ret <32 x i1> %0
}
-define <8 x i1> @whilewr_32_expand(i64 %a, i64 %b) {
+define <8 x i1> @whilewr_32_expand(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_expand:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs x8, x1, x0
@@ -233,11 +234,11 @@ define <8 x i1> @whilewr_32_expand(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 4)
ret <8 x i1> %0
}
-define <16 x i1> @whilewr_32_expand2(i64 %a, i64 %b) {
+define <16 x i1> @whilewr_32_expand2(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_expand2:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs x8, x1, x0
@@ -251,11 +252,11 @@ define <16 x i1> @whilewr_32_expand2(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 4)
ret <16 x i1> %0
}
-define <32 x i1> @whilewr_32_expand3(i64 %a, i64 %b) {
+define <32 x i1> @whilewr_32_expand3(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_expand3:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs x9, x1, x0
@@ -273,21 +274,21 @@ define <32 x i1> @whilewr_32_expand3(i64 %a, i64 %b) {
; CHECK-NEXT: mov z2.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: zip1 v1.16b, v1.16b, v3.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT: str h0, [x8, #2]
; CHECK-NEXT: str h1, [x8]
; CHECK-NEXT: ret
entry:
- %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1(ptr %a, ptr %b, i64 4)
ret <32 x i1> %0
}
-define <4 x i1> @whilewr_64_expand(i64 %a, i64 %b) {
+define <4 x i1> @whilewr_64_expand(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_expand:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs x8, x1, x0
@@ -301,11 +302,11 @@ define <4 x i1> @whilewr_64_expand(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 %a, i64 %b, i64 8)
+ %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr %a, ptr %b, i64 8)
ret <4 x i1> %0
}
-define <8 x i1> @whilewr_64_expand2(i64 %a, i64 %b) {
+define <8 x i1> @whilewr_64_expand2(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_expand2:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs x8, x1, x0
@@ -319,11 +320,11 @@ define <8 x i1> @whilewr_64_expand2(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 8)
+ %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 8)
ret <8 x i1> %0
}
-define <16 x i1> @whilewr_64_expand3(i64 %a, i64 %b) {
+define <16 x i1> @whilewr_64_expand3(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_expand3:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs x8, x1, x0
@@ -337,11 +338,11 @@ define <16 x i1> @whilewr_64_expand3(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 8)
+ %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 8)
ret <16 x i1> %0
}
-define <32 x i1> @whilewr_64_expand4(i64 %a, i64 %b) {
+define <32 x i1> @whilewr_64_expand4(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_expand4:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs x9, x1, x0
@@ -359,21 +360,21 @@ define <32 x i1> @whilewr_64_expand4(i64 %a, i64 %b) {
; CHECK-NEXT: mov z2.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: zip1 v1.16b, v1.16b, v3.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT: str h0, [x8, #2]
; CHECK-NEXT: str h1, [x8]
; CHECK-NEXT: ret
entry:
- %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1.i64(i64 %a, i64 %b, i64 8)
+ %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1(ptr %a, ptr %b, i64 8)
ret <32 x i1> %0
}
-define <9 x i1> @whilewr_8_widen(i64 %a, i64 %b) {
+define <9 x i1> @whilewr_8_widen(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8_widen:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.b, x0, x1
@@ -401,11 +402,11 @@ define <9 x i1> @whilewr_8_widen(i64 %a, i64 %b) {
; CHECK-NEXT: strh w9, [x8]
; CHECK-NEXT: ret
entry:
- %0 = call <9 x i1> @llvm.loop.dependence.war.mask.v9i1.i64(i64 %a, i64 %b, i64 1)
+ %0 = call <9 x i1> @llvm.loop.dependence.war.mask.v9i1(ptr %a, ptr %b, i64 1)
ret <9 x i1> %0
}
-define <7 x i1> @whilewr_16_widen(i64 %a, i64 %b) {
+define <7 x i1> @whilewr_16_widen(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16_widen:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.h, x0, x1
@@ -420,11 +421,11 @@ define <7 x i1> @whilewr_16_widen(i64 %a, i64 %b) {
; CHECK-NEXT: umov w6, v0.b[6]
; CHECK-NEXT: ret
entry:
- %0 = call <7 x i1> @llvm.loop.dependence.war.mask.v7i1.i64(i64 %a, i64 %b, i64 2)
+ %0 = call <7 x i1> @llvm.loop.dependence.war.mask.v7i1(ptr %a, ptr %b, i64 2)
ret <7 x i1> %0
}
-define <3 x i1> @whilewr_32_widen(i64 %a, i64 %b) {
+define <3 x i1> @whilewr_32_widen(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_widen:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.s, x0, x1
@@ -435,11 +436,11 @@ define <3 x i1> @whilewr_32_widen(i64 %a, i64 %b) {
; CHECK-NEXT: umov w2, v0.h[2]
; CHECK-NEXT: ret
entry:
- %0 = call <3 x i1> @llvm.loop.dependence.war.mask.v3i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <3 x i1> @llvm.loop.dependence.war.mask.v3i1(ptr %a, ptr %b, i64 4)
ret <3 x i1> %0
}
-define <16 x i1> @whilewr_badimm(i64 %a, i64 %b) {
+define <16 x i1> @whilewr_badimm(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_badimm:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov x8, #6148914691236517205 // =0x5555555555555555
@@ -454,93 +455,93 @@ define <16 x i1> @whilewr_badimm(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 3)
+ %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 3)
ret <16 x i1> %0
}
; Scalarizing <1 x i1> types
-define <1 x i1> @whilewr_8_scalarize(i64 %a, i64 %b) {
+define <1 x i1> @whilewr_8_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8_scalarize:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w0, #1 // =0x1
; CHECK-NEXT: ret
entry:
- %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1.i64(i64 %a, i64 %b, i64 1)
+ %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 1)
ret <1 x i1> %0
}
-define <1 x i1> @whilewr_16_scalarize(i64 %a, i64 %b) {
+define <1 x i1> @whilewr_16_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16_scalarize:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w0, #1 // =0x1
; CHECK-NEXT: ret
entry:
- %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1.i64(i64 %a, i64 %b, i64 2)
+ %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 2)
ret <1 x i1> %0
}
-define <1 x i1> @whilewr_32_scalarize(i64 %a, i64 %b) {
+define <1 x i1> @whilewr_32_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_scalarize:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w0, #1 // =0x1
; CHECK-NEXT: ret
entry:
- %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 4)
ret <1 x i1> %0
}
-define <1 x i1> @whilewr_64_scalarize(i64 %a, i64 %b) {
+define <1 x i1> @whilewr_64_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_scalarize:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w0, #1 // =0x1
; CHECK-NEXT: ret
entry:
- %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1.i64(i64 %a, i64 %b, i64 8)
+ %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 8)
ret <1 x i1> %0
}
-define <1 x i1> @whilerw_8_scalarize(i64 %a, i64 %b) {
+define <1 x i1> @whilerw_8_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_8_scalarize:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w0, #1 // =0x1
; CHECK-NEXT: ret
entry:
- %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1.i64(i64 %a, i64 %b, i64 1)
+ %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 1)
ret <1 x i1> %0
}
-define <1 x i1> @whilerw_16_scalarize(i64 %a, i64 %b) {
+define <1 x i1> @whilerw_16_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_16_scalarize:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w0, #1 // =0x1
; CHECK-NEXT: ret
entry:
- %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1.i64(i64 %a, i64 %b, i64 2)
+ %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 2)
ret <1 x i1> %0
}
-define <1 x i1> @whilerw_32_scalarize(i64 %a, i64 %b) {
+define <1 x i1> @whilerw_32_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_32_scalarize:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w0, #1 // =0x1
; CHECK-NEXT: ret
entry:
- %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 4)
ret <1 x i1> %0
}
-define <1 x i1> @whilerw_64_scalarize(i64 %a, i64 %b) {
+define <1 x i1> @whilerw_64_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_64_scalarize:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w0, #1 // =0x1
; CHECK-NEXT: ret
entry:
- %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1.i64(i64 %a, i64 %b, i64 8)
+ %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 8)
ret <1 x i1> %0
}
-define <8 x i1> @whilewr_extract_v8i1(i64 %a, i64 %b) {
+define <8 x i1> @whilewr_extract_v8i1(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_extract_v8i1:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.b, x0, x1
@@ -548,11 +549,11 @@ define <8 x i1> @whilewr_extract_v8i1(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 1)
+ %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 1)
ret <8 x i1> %0
}
-define <4 x i1> @whilewr_extract_v4i1(i64 %a, i64 %b) {
+define <4 x i1> @whilewr_extract_v4i1(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_extract_v4i1:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.b, x0, x1
@@ -561,11 +562,11 @@ define <4 x i1> @whilewr_extract_v4i1(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 %a, i64 %b, i64 1)
+ %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr %a, ptr %b, i64 1)
ret <4 x i1> %0
}
-define <2 x i1> @whilewr_extract_v2i1(i64 %a, i64 %b) {
+define <2 x i1> @whilewr_extract_v2i1(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_extract_v2i1:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: whilewr p0.s, x0, x1
@@ -573,34 +574,6 @@ define <2 x i1> @whilewr_extract_v2i1(i64 %a, i64 %b) {
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
entry:
- %0 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1.i64(i64 %a, i64 %b, i64 4)
+ %0 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1(ptr %a, ptr %b, i64 4)
ret <2 x i1> %0
}
-
-define <16 x i1> @war_mask_i16(i16 %a, i16 %b) {
-; CHECK-LABEL: war_mask_i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: and w8, w0, #0xffff
-; CHECK-NEXT: and w9, w1, #0xffff
-; CHECK-NEXT: whilewr p0.b, x8, x9
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT: ret
-entry:
- %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i16(i16 %a, i16 %b, i16 1)
- ret <16 x i1> %0
-}
-
-define <16 x i1> @raw_mask_i16(i16 %a, i16 %b) {
-; CHECK-LABEL: raw_mask_i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: and w8, w0, #0xffff
-; CHECK-NEXT: and w9, w1, #0xffff
-; CHECK-NEXT: whilerw p0.b, x8, x9
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT: ret
-entry:
- %0 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1.i16(i16 %a, i16 %b, i16 1)
- ret <16 x i1> %0
-}
diff --git a/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll b/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
index 926fe2d4a6515..426a95718d96a 100644
--- a/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
+++ b/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
@@ -338,10 +338,10 @@ define i16 @combine_setcc_slt_add_nsw(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: cmgt v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI20_0]
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: umov w0, v0.h[0]
; CHECK-NEXT: ret
%add0 = sub nsw <16 x i8> %a, %b
%cmp1 = icmp slt <16 x i8> %add0, zeroinitializer
@@ -356,10 +356,10 @@ define i16 @combine_setcc_sgt_add_nsw(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: cmgt v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI21_0]
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: umov w0, v0.h[0]
; CHECK-NEXT: ret
%add0 = sub nsw <16 x i8> %a, %b
%cmp1 = icmp sgt <16 x i8> %add0, zeroinitializer
@@ -374,10 +374,10 @@ define i16 @combine_setcc_sle_add_nsw(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: cmge v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI22_0]
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: umov w0, v0.h[0]
; CHECK-NEXT: ret
%add0 = sub nsw <16 x i8> %a, %b
%cmp1 = icmp sle <16 x i8> %add0, zeroinitializer
@@ -391,10 +391,10 @@ define i16 @combine_setcc_sge_add_nsw(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: cmge v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI23_0]
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: umov w0, v0.h[0]
; CHECK-NEXT: ret
%add0 = sub nsw <16 x i8> %a, %b
%cmp1 = icmp sge <16 x i8> %add0, zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
index 730dfed5ff228..da917119c6b25 100644
--- a/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
@@ -15,14 +15,14 @@ define { <16 x i8>, <16 x i8> } @foo_ld2_v16i8(<16 x i1> %mask, ptr %p) {
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v1.16b, v1.16b, v2.16b
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: addv h1, v1.8h
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: fmov w9, s1
-; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: umov w9, v1.h[0]
+; CHECK-NEXT: umov w8, v0.h[0]
; CHECK-NEXT: bfi w8, w9, #16, #16
; CHECK-NEXT: tbz w8, #0, .LBB0_2
; CHECK-NEXT: // %bb.1: // %cond.load
@@ -236,10 +236,10 @@ define { <8 x i16>, <8 x i16> } @foo_ld2_v8i16(<8 x i1> %mask, ptr %p) {
; CHECK-NEXT: shl v0.16b, v0.16b, #7
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: umov w8, v0.h[0]
; CHECK-NEXT: tbz w8, #0, .LBB1_2
; CHECK-NEXT: // %bb.1: // %cond.load
; CHECK-NEXT: ldr h1, [x0]
diff --git a/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
index ac1db2bc56bbd..972be0fca2089 100644
--- a/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
@@ -15,14 +15,14 @@ define void @foo_st2_v16i8(<16 x i1> %mask, <16 x i8> %val1, <16 x i8> %val2, pt
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
; CHECK-NEXT: and v3.16b, v3.16b, v4.16b
; CHECK-NEXT: and v0.16b, v0.16b, v4.16b
-; CHECK-NEXT: ext v4.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT: ext v5.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v3.16b, v3.16b, v4.16b
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v5.16b
-; CHECK-NEXT: addv h3, v3.8h
-; CHECK-NEXT: addv h0, v0.8h
-; CHECK-NEXT: fmov w9, s3
-; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: umov w9, v3.h[0]
+; CHECK-NEXT: umov w8, v0.h[0]
; CHECK-NEXT: zip1 v0.16b, v1.16b, v2.16b
; CHECK-NEXT: bfi w8, w9, #16, #16
; CHECK-NEXT: tbnz w8, #0, .LBB0_33
@@ -234,11 +234,11 @@ define void @foo_st2_v8i16(<8 x i1> %mask, <8 x i16> %val1, <8 x i16> %val2, ptr
; CHECK-NEXT: shl v0.16b, v0.16b, #7
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
; CHECK-NEXT: and v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: addv h3, v0.8h
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: umov w8, v0.h[0]
; CHECK-NEXT: zip1 v0.8h, v1.8h, v2.8h
-; CHECK-NEXT: fmov w8, s3
; CHECK-NEXT: tbnz w8, #0, .LBB1_17
; CHECK-NEXT: // %bb.1: // %else
; CHECK-NEXT: tbnz w8, #1, .LBB1_18
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-expandloads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-expandloads.ll
index 8789167e611e4..2aa205b60aa98 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-expandloads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-expandloads.ll
@@ -357,11 +357,11 @@ define void @masked_load_v16f32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]
; VBITS_GE_256-NEXT: ldr q0, [x8, :lo12:.LCPI4_0]
; VBITS_GE_256-NEXT: and v0.16b, v1.16b, v0.16b
-; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_256-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: addv h0, v0.8h
-; VBITS_GE_256-NEXT: fmov w9, s0
-; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w9, v0.h[0]
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
; VBITS_GE_256-NEXT: tbz w9, #0, .LBB4_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldr s0, [x0], #4
@@ -5338,6 +5338,8 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: adrp x10, .LCPI9_1
+; VBITS_GE_256-NEXT: add x10, x10, :lo12:.LCPI9_1
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
@@ -5354,15 +5356,13 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b
; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]
; VBITS_GE_256-NEXT: ldr q0, [x8, :lo12:.LCPI9_0]
-; VBITS_GE_256-NEXT: adrp x8, .LCPI9_1
-; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI9_1
; VBITS_GE_256-NEXT: and v0.16b, v1.16b, v0.16b
-; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_256-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: addv h1, v0.8h
-; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x8]
-; VBITS_GE_256-NEXT: fmov w9, s1
-; VBITS_GE_256-NEXT: fmov w8, s1
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w9, v0.h[0]
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x10]
; VBITS_GE_256-NEXT: tbz w9, #0, .LBB9_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rw { z2.s }, p1/z, [x0]
@@ -7627,10 +7627,10 @@ define void @masked_load_sext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI14_0]
; VBITS_GE_256-NEXT: cmeq v0.16b, v0.16b, #0
; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_256-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: addv h0, v0.8h
-; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
; VBITS_GE_256-NEXT: tbz w8, #0, .LBB14_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrb w9, [x0], #1
@@ -7731,10 +7731,10 @@ define void @masked_load_sext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI14_0]
; VBITS_GE_512-NEXT: cmeq v0.16b, v0.16b, #0
; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
-; VBITS_GE_512-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_512-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_512-NEXT: addv h0, v0.8h
-; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
; VBITS_GE_512-NEXT: tbz w8, #0, .LBB14_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldrb w9, [x0], #1
@@ -9833,10 +9833,10 @@ define void @masked_load_zext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI20_0]
; VBITS_GE_256-NEXT: cmeq v0.16b, v0.16b, #0
; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_256-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: addv h0, v0.8h
-; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
; VBITS_GE_256-NEXT: tbz w8, #0, .LBB20_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrb w9, [x0], #1
@@ -9937,10 +9937,10 @@ define void @masked_load_zext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI20_0]
; VBITS_GE_512-NEXT: cmeq v0.16b, v0.16b, #0
; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
-; VBITS_GE_512-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_512-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_512-NEXT: addv h0, v0.8h
-; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
; VBITS_GE_512-NEXT: tbz w8, #0, .LBB20_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldrb w9, [x0], #1
@@ -12030,11 +12030,11 @@ define void @masked_load_sext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]
; VBITS_GE_256-NEXT: ldr q0, [x8, :lo12:.LCPI26_0]
; VBITS_GE_256-NEXT: and v0.16b, v1.16b, v0.16b
-; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_256-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: addv h0, v0.8h
-; VBITS_GE_256-NEXT: fmov w9, s0
-; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w9, v0.h[0]
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
; VBITS_GE_256-NEXT: tbz w9, #0, .LBB26_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrb w9, [x0], #1
@@ -12532,11 +12532,11 @@ define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]
; VBITS_GE_256-NEXT: ldr q0, [x8, :lo12:.LCPI28_0]
; VBITS_GE_256-NEXT: and v0.16b, v1.16b, v0.16b
-; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_256-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: addv h0, v0.8h
-; VBITS_GE_256-NEXT: fmov w9, s0
-; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w9, v0.h[0]
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
; VBITS_GE_256-NEXT: tbz w9, #0, .LBB28_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rh { z0.h }, p1/z, [x0]
@@ -14394,11 +14394,11 @@ define void @masked_load_zext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]
; VBITS_GE_256-NEXT: ldr q0, [x8, :lo12:.LCPI32_0]
; VBITS_GE_256-NEXT: and v0.16b, v1.16b, v0.16b
-; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_256-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: addv h0, v0.8h
-; VBITS_GE_256-NEXT: fmov w9, s0
-; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w9, v0.h[0]
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
; VBITS_GE_256-NEXT: tbz w9, #0, .LBB32_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrb w9, [x0], #1
@@ -14896,11 +14896,11 @@ define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]
; VBITS_GE_256-NEXT: ldr q0, [x8, :lo12:.LCPI34_0]
; VBITS_GE_256-NEXT: and v0.16b, v1.16b, v0.16b
-; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; VBITS_GE_256-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; VBITS_GE_256-NEXT: addv h0, v0.8h
-; VBITS_GE_256-NEXT: fmov w9, s0
-; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w9, v0.h[0]
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
; VBITS_GE_256-NEXT: tbz w9, #0, .LBB34_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rh { z0.h }, p1/z, [x0]
diff --git a/llvm/test/CodeGen/AArch64/sve-mask-partition.ll b/llvm/test/CodeGen/AArch64/sve-mask-partition.ll
index bf5151380e99e..ec31054e7f12a 100644
--- a/llvm/test/CodeGen/AArch64/sve-mask-partition.ll
+++ b/llvm/test/CodeGen/AArch64/sve-mask-partition.ll
@@ -381,12 +381,12 @@ define <32 x i1> @mask_exclude_active_v32(<32 x i1> %mask.in) {
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v1.16b, v1.16b, v2.16b
-; CHECK-NEXT: zip1 v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: addv h1, v1.8h
-; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT: str h1, [x8, #2]
; CHECK-NEXT: str h0, [x8]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index b84c916f7b595..2142f898bf9e7 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -17,10 +17,10 @@ define i16 @convert_to_bitmask16(<16 x i8> %vec) {
; CHECK-SD-NEXT: cmeq.16b v0, v0, #0
; CHECK-SD-NEXT: ldr q1, [x8, lCPI0_0 at PAGEOFF]
; CHECK-SD-NEXT: bic.16b v0, v1, v0
-; CHECK-SD-NEXT: ext.16b v1, v0, v0, #8
-; CHECK-SD-NEXT: zip1.16b v0, v0, v1
-; CHECK-SD-NEXT: addv.8h h0, v0
-; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
+; CHECK-SD-NEXT: umov.h w0, v0[0]
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: convert_to_bitmask16:
@@ -521,10 +521,10 @@ define i16 @convert_to_bitmask_without_knowing_type(<16 x i1> %vec) {
; CHECK-SD-NEXT: ldr q1, [x8, lCPI10_0 at PAGEOFF]
; CHECK-SD-NEXT: cmlt.16b v0, v0, #0
; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: ext.16b v1, v0, v0, #8
-; CHECK-SD-NEXT: zip1.16b v0, v0, v1
-; CHECK-SD-NEXT: addv.8h h0, v0
-; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
+; CHECK-SD-NEXT: umov.h w0, v0[0]
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: convert_to_bitmask_without_knowing_type:
@@ -996,9 +996,9 @@ define <2 x i8> @vector_to_vector_cast(<16 x i1> %arg) nounwind {
; CHECK-SD-NEXT: ldr q1, [x8, lCPI20_0 at PAGEOFF]
; CHECK-SD-NEXT: cmlt.16b v0, v0, #0
; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: ext.16b v1, v0, v0, #8
-; CHECK-SD-NEXT: zip1.16b v0, v0, v1
-; CHECK-SD-NEXT: addv.8h h0, v0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
+; CHECK-SD-NEXT: addp.16b v0, v0, v0
; CHECK-SD-NEXT: ushll.8h v0, v0, #0
; CHECK-SD-NEXT: ushll.4s v0, v0, #0
; CHECK-SD-NEXT: ; kill: def $d0 killed $d0 killed $q0
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll
index f3c485641bff3..a234b2bc2f385 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll
@@ -11,9 +11,9 @@ define void @store_16_elements(<16 x i8> %vec, ptr %out) {
; CHECK-NEXT: Lloh1:
; CHECK-NEXT: ldr q1, [x8, lCPI0_0 at PAGEOFF]
; CHECK-NEXT: bic.16b v0, v1, v0
-; CHECK-NEXT: ext.16b v1, v0, v0, #8
-; CHECK-NEXT: zip1.16b v0, v0, v1
-; CHECK-NEXT: addv.8h h0, v0
+; CHECK-NEXT: addp.16b v0, v0, v0
+; CHECK-NEXT: addp.16b v0, v0, v0
+; CHECK-NEXT: addp.16b v0, v0, v0
; CHECK-NEXT: str h0, [x0]
; CHECK-NEXT: ret
; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh1
More information about the llvm-commits
mailing list