[llvm] a631bc1 - [IR][REVEC] Define llvm.vector.repeat intrinsic (#208212)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 03:17:58 PDT 2026
Author: Gaëtan Bossu
Date: 2026-09-25T11:17:46+01:00
New Revision: a631bc15c5445e48124a06a884ecf4bf9ec491e2
URL: https://github.com/llvm/llvm-project/commit/a631bc15c5445e48124a06a884ecf4bf9ec491e2
DIFF: https://github.com/llvm/llvm-project/commit/a631bc15c5445e48124a06a884ecf4bf9ec491e2.diff
LOG: [IR][REVEC] Define llvm.vector.repeat intrinsic (#208212)
This intrinsic is used to repeat a <N x Ty> fixed-length vector `vscale`
times to form a <vscale x N x Ty> scalable vector. The patch adds
legalisation support and ISel for AArch64.
This intrinsic is required to support re-vectorisation of fixed-length
vectors to scalable ones, as shufflevector instructions do not support
scalable vectors. Re-vectorisation was discussed in this RFC:
https://discourse.llvm.org/t/rfc-re-vectorisation-to-wider-vectors-in-loopvectorizer/91071
Assisted-by: codex (tests)
Added:
llvm/test/CodeGen/AArch64/sve-vector-repeat.ll
llvm/test/Verifier/vector-repeat-intrinsic-invalid.ll
llvm/test/Verifier/vector-repeat-intrinsic-valid.ll
Modified:
llvm/docs/LangRef.md
llvm/include/llvm/CodeGen/ISDOpcodes.h
llvm/include/llvm/IR/Intrinsics.td
llvm/include/llvm/Target/TargetSelectionDAG.td
llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.h
llvm/lib/Target/AArch64/SVEInstrFormats.td
llvm/unittests/CodeGen/SelectionDAGNodeConstructionTest.cpp
Removed:
################################################################################
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 43cef3fada731..9d1b787c39ec5 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -20810,6 +20810,28 @@ runtime, then the result vector is a {ref}`poison value <poisonvalues>`. The
`idx` parameter must be a vector index constant type (for most targets this
will be an integer pointer type).
+#### '`llvm.vector.repeat`' Intrinsic
+
+##### Syntax:
+This is an overloaded intrinsic.
+
+```
+declare <vscale x 16 x i8> @llvm.vector.repeat.nxv16i8.v16i8(<16 x i8> %vec)
+```
+
+##### Overview:
+
+The '`llvm.vector.repeat.*`' intrinsic repeatedly copies the elements of the
+source fixed-length vector, in order, until the result scalable vector is
+filled. For example, repeating `<A, B>` produces a scalable vector containing
+`vscale` copies of `<A, B>`.
+
+##### Arguments:
+
+The argument must be a fixed-length vector (i.e. `<N x Ty>`) and the result a
+scalable vector that is exactly `vscale` times longer (i.e.
+`<vscale x N x Ty>`).
+
#### '`llvm.vector.reverse`' Intrinsic
##### Syntax:
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 1af501d4c8dd0..953201a26a3e7 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -637,6 +637,11 @@ enum NodeType {
/// Result[J] = EXTRACT_SUBVECTOR(Interleaved, J * getVectorMinNumElements())
VECTOR_INTERLEAVE,
+ /// VECTOR_REPEAT(FIXED_LENGTH_VECTOR)
+ /// Repeatedly copies the elements of the source fixed-length vector to fill a
+ /// scalable vector with the same minimum element count.
+ VECTOR_REPEAT,
+
/// VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR,
/// whose elements are shuffled using the following algorithm:
/// RESULT[i] = VECTOR[VECTOR.ElementCount - 1 - i]
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index fa1ab702873d2..52be391132b7e 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2811,6 +2811,12 @@ foreach n = 2...8 in {
[IntrNoMem, IntrSpeculatable]>;
}
+// vector_repeat( SrcVector )
+// Repeat a fixed-length vector to fill its "vscale x" equivalent.
+def int_vector_repeat : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [llvm_anyvector_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+
//===-------------- Intrinsics to perform partial reduction ---------------===//
def int_vector_partial_reduce_add : DefaultAttrsIntrinsic<[LLVMMatchType<0>],
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index b0fd9c733e5b6..22fe30fea4fea 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -936,6 +936,10 @@ def vector_insert_subvec : SDNode<"ISD::INSERT_SUBVECTOR",
def extract_subvector : SDNode<"ISD::EXTRACT_SUBVECTOR", SDTSubVecExtract, []>;
def insert_subvector : SDNode<"ISD::INSERT_SUBVECTOR", SDTSubVecInsert, []>;
+def vector_repeat : SDNode<"ISD::VECTOR_REPEAT",
+ SDTypeProfile<1, 1, [SDTCisVec<1>, SDTCisVec<0>]>,
+ []>;
+
def find_last_active
: SDNode<"ISD::VECTOR_FIND_LAST_ACTIVE",
SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisVec<1>]>, []>;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index a90bb06fb424d..5b6329c2560eb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -127,6 +127,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_SPLICE_RIGHT:
Res = PromoteIntRes_VECTOR_SPLICE(N);
break;
+ case ISD::VECTOR_REPEAT:
+ Res = PromoteIntRes_VECTOR_REPEAT(N);
+ break;
case ISD::VECTOR_INTERLEAVE:
case ISD::VECTOR_DEINTERLEAVE:
Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
@@ -2157,6 +2160,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
case ISD::PARTIAL_REDUCE_SUMLA:
Res = PromoteIntOp_PARTIAL_REDUCE_MLA(N);
break;
+ case ISD::VECTOR_REPEAT:
+ Res = PromoteIntOp_VECTOR_REPEAT(N);
+ break;
case ISD::LOOP_DEPENDENCE_RAW_MASK:
case ISD::LOOP_DEPENDENCE_WAR_MASK:
Res = PromoteIntOp_LOOP_DEPENDENCE_MASK(N);
@@ -3018,6 +3024,17 @@ SDValue DAGTypeLegalizer::PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
}
+SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_REPEAT(SDNode *N) {
+ SDLoc DL(N);
+ SDValue Src = GetPromotedInteger(N->getOperand(0));
+ EVT SrcVT = Src.getValueType();
+ EVT OrigVT = N->getValueType(0);
+ EVT NewVT = OrigVT.changeVectorElementType(*DAG.getContext(),
+ SrcVT.getVectorElementType());
+ SDValue Res = DAG.getNode(ISD::VECTOR_REPEAT, DL, NewVT, Src);
+ return DAG.getNode(ISD::TRUNCATE, DL, OrigVT, Res);
+}
+
//===----------------------------------------------------------------------===//
// Integer Result Expansion
//===----------------------------------------------------------------------===//
@@ -6106,6 +6123,18 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
return DAG.getNode(N->getOpcode(), dl, OutVT, V0, V1, N->getOperand(2));
}
+SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REPEAT(SDNode *N) {
+ SDLoc DL(N);
+
+ EVT OutVT = N->getValueType(0);
+ EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
+ EVT NInVT = N->getOperand(0).getValueType().changeVectorElementType(
+ *DAG.getContext(), NOutVT.getVectorElementType());
+
+ SDValue Op = DAG.getNode(ISD::ANY_EXTEND, DL, NInVT, N->getOperand(0));
+ return DAG.getNode(N->getOpcode(), DL, NOutVT, Op);
+}
+
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
SDLoc DL(N);
unsigned Factor = N->getNumOperands();
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index c9583d99ddbf4..b638d00673490 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -286,6 +286,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_VECTOR_REVERSE(SDNode *N);
SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
SDValue PromoteIntRes_VECTOR_SPLICE(SDNode *N);
+ SDValue PromoteIntRes_VECTOR_REPEAT(SDNode *N);
SDValue PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N);
SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
SDValue PromoteIntRes_ScalarOp(SDNode *N);
@@ -419,6 +420,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N);
SDValue PromoteIntOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N);
+ SDValue PromoteIntOp_VECTOR_REPEAT(SDNode *N);
SDValue PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N);
SDValue PromoteIntOp_MaskedBinOp(SDNode *N, unsigned OpNo);
@@ -952,6 +954,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitVecRes_VECTOR_REPEAT(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
SDValue &Hi);
@@ -1100,6 +1103,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
bool WidenVectorOperand(SDNode *N, unsigned OpNo);
SDValue WidenVecOp_BITCAST(SDNode *N);
SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
+ SDValue WidenVecOp_VECTOR_REPEAT(SDNode *N);
SDValue WidenVecOp_EXTEND(SDNode *N);
SDValue WidenVecOp_CMP(SDNode *N);
SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 56ad7c65b48d1..2e78858d422b9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1449,6 +1449,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SETCC:
SplitVecRes_SETCC(N, Lo, Hi);
break;
+ case ISD::VECTOR_REPEAT:
+ SplitVecRes_VECTOR_REPEAT(N, Lo, Hi);
+ break;
case ISD::VECTOR_REVERSE:
SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
break;
@@ -3485,6 +3488,31 @@ void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
}
+void DAGTypeLegalizer::SplitVecRes_VECTOR_REPEAT(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ EVT VT = N->getValueType(0);
+ SDValue Src = N->getOperand(0);
+ auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
+ assert(LoVT == HiVT && "Expected equal split types");
+
+ // Use smaller even/odd source vectors so their broadcasts can be
+ // reinterleaved in the original lane order for every value of vscale.
+ SDLoc DL(N);
+ auto [SrcLo, SrcHi] = DAG.SplitVector(Src, DL);
+ EVT SplitSrcVT = SrcLo.getValueType();
+ SDValue Deinterleaved =
+ DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
+ DAG.getVTList(SplitSrcVT, SplitSrcVT), SrcLo, SrcHi);
+ SDValue Even =
+ DAG.getNode(ISD::VECTOR_REPEAT, DL, LoVT, Deinterleaved.getValue(0));
+ SDValue Odd =
+ DAG.getNode(ISD::VECTOR_REPEAT, DL, LoVT, Deinterleaved.getValue(1));
+ SDValue Interleaved = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
+ DAG.getVTList(LoVT, LoVT), Even, Odd);
+ Lo = Interleaved.getValue(0);
+ Hi = Interleaved.getValue(1);
+}
+
void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
SDValue &Hi) {
SDValue InLo, InHi;
@@ -7809,6 +7837,9 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
Res = WidenVecOp_FAKE_USE(N);
break;
case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
+ case ISD::VECTOR_REPEAT:
+ Res = WidenVecOp_VECTOR_REPEAT(N);
+ break;
case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
@@ -8263,6 +8294,32 @@ SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
return DAG.getBuildVector(VT, dl, Ops);
}
+SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_REPEAT(SDNode *N) {
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+ SDValue Src = N->getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ EVT WidenedSrcVT = TLI.getTypeToTransformTo(*DAG.getContext(), SrcVT);
+
+ if (!WidenedSrcVT.getVectorElementCount().hasKnownScalarFactor(
+ SrcVT.getVectorElementCount()))
+ report_fatal_error(
+ "Cannot widen VECTOR_REPEAT operand to an ElementCount that's not "
+ "a known scalar multiple of the input ElementCount.");
+
+ // Repeat the original source because the extra lanes of its widened value
+ // are unspecified.
+ unsigned NumConcat =
+ WidenedSrcVT.getVectorNumElements() / SrcVT.getVectorNumElements();
+ SmallVector<SDValue, 8> Ops(NumConcat, Src);
+ SDValue WidenedSrc = DAG.getNode(ISD::CONCAT_VECTORS, DL, WidenedSrcVT, Ops);
+ EVT WidenedVT = VT.changeVectorElementCount(
+ *DAG.getContext(),
+ ElementCount::getScalable(WidenedSrcVT.getVectorNumElements()));
+ SDValue Widened = DAG.getNode(ISD::VECTOR_REPEAT, DL, WidenedVT, WidenedSrc);
+ return DAG.getExtractSubvector(DL, VT, Widened, 0);
+}
+
SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
EVT VT = N->getValueType(0);
SDValue SubVec = N->getOperand(1);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 8c67b0449ddb6..541c378a2eda1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -7447,6 +7447,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
if (N1.getValueType().getScalarType() == MVT::i1)
return getNode(ISD::VECREDUCE_AND, DL, VT, N1);
break;
+ case ISD::VECTOR_REPEAT:
+ assert(N1.getValueType().isFixedLengthVector() &&
+ "VECTOR_REPEAT requires a fixed-length vector operand");
+ assert(VT.isScalableVector() &&
+ "VECTOR_REPEAT requires a scalable vector result");
+ assert(N1.getValueType().getVectorNumElements() ==
+ VT.getVectorMinNumElements() &&
+ "VECTOR_REPEAT operand and result element counts must match");
+ if (VT.getVectorMinNumElements() == 1)
+ return getSplatVector(
+ VT, DL, getExtractVectorElt(DL, VT.getVectorElementType(), N1, 0));
+ break;
case ISD::SPLAT_VECTOR:
assert(VT.isVector() && "Wrong return type!");
// FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 7d185031d7d37..05f98cea6bbdb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8649,6 +8649,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::vector_deinterleave8:
visitVectorDeinterleave(I, 8);
return;
+ case Intrinsic::vector_repeat: {
+ SDValue Vec = getValue(I.getOperand(0));
+ EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+ setValue(&I, DAG.getNode(ISD::VECTOR_REPEAT, sdl, ResultVT, Vec));
+ return;
+ }
case Intrinsic::experimental_vector_compress:
setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,
getValue(I.getArgOperand(0)).getValueType(),
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 8bc1707b47b31..bfb07ba1faeaf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -361,6 +361,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::EXTRACT_SUBVECTOR: return "extract_subvector";
case ISD::VECTOR_DEINTERLEAVE: return "vector_deinterleave";
case ISD::VECTOR_INTERLEAVE: return "vector_interleave";
+ case ISD::VECTOR_REPEAT: return "vector_repeat";
case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector";
case ISD::VECTOR_SHUFFLE: return "vector_shuffle";
case ISD::VECTOR_SPLICE_LEFT: return "vector_splice_left";
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1440b95896474..7af5cf037373f 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7057,6 +7057,23 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
}
break;
}
+ case Intrinsic::vector_repeat: {
+ auto *ResultTy = dyn_cast<ScalableVectorType>(Call.getType());
+ auto *ArgTy = dyn_cast<FixedVectorType>(Call.getArgOperand(0)->getType());
+
+ Check(ArgTy, "vector_repeat argument must be a fixed-length vector.",
+ &Call);
+ Check(ResultTy, "vector_repeat result must be a scalable vector.", &Call);
+ Check(ResultTy->getElementType() == ArgTy->getElementType(),
+ "vector_repeat argument and result must have the same element "
+ "type.",
+ &Call);
+ Check(ArgTy->getNumElements() == ResultTy->getMinNumElements(),
+ "vector_repeat argument and result must have the same minimum "
+ "element count.",
+ &Call);
+ break;
+ }
case Intrinsic::vector_insert: {
Value *Vec = Call.getArgOperand(0);
Value *SubVec = Call.getArgOperand(1);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d048d0eb7acf4..395d586bd89eb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1991,6 +1991,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Custom);
}
+ // Direct patterns exist for quad broadcasts.
+ for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64,
+ MVT::nxv8f16, MVT::nxv4f32, MVT::nxv2f64, MVT::nxv8bf16})
+ setOperationAction(ISD::VECTOR_REPEAT, VT, Legal);
+
+ // VECTOR_REPEAT to legal unpacked SVE types require explicit unpacking to
+ // add spacing between elements.
+ for (auto VT : {MVT::nxv4f16, MVT::nxv2f32, MVT::nxv4bf16})
+ setOperationAction(ISD::VECTOR_REPEAT, VT, Custom);
+
if (Subtarget->hasSVEB16B16() &&
Subtarget->isNonStreamingSVEorSME2Available()) {
// Note: Use SVE for bfloat16 operations when +sve-b16b16 is available.
@@ -8873,6 +8883,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerEXTEND_VECTOR_INREG(Op, DAG);
case ISD::ZERO_EXTEND_VECTOR_INREG:
return LowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
+ case ISD::VECTOR_REPEAT:
+ return LowerVECTOR_REPEAT(Op, DAG);
case ISD::VECTOR_SHUFFLE:
return LowerVECTOR_SHUFFLE(Op, DAG);
case ISD::SPLAT_VECTOR:
@@ -17837,6 +17849,23 @@ SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
return SDValue();
}
+SDValue AArch64TargetLowering::LowerVECTOR_REPEAT(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ SDValue Src = Op.getOperand(0);
+ EVT VT = Op.getValueType();
+ EVT SrcVT = Src.getValueType();
+ assert(SrcVT.is64BitVector() && "Expected 64bit source!");
+
+ // Repeat into a packed container before extracting the low lanes, which
+ // places the result elements at the spacing required by the unpacked type.
+ SDValue SrcAsScalar =
+ DAG.getExtractVectorElt(DL, MVT::i64, DAG.getBitcast(MVT::v1i64, Src), 0);
+ SDValue Splat = DAG.getSplat(MVT::nxv2i64, DL, SrcAsScalar);
+ EVT PackedVT = VT.getDoubleNumVectorElementsVT(*DAG.getContext());
+ return DAG.getExtractSubvector(DL, VT, DAG.getBitcast(PackedVT, Splat), 0);
+}
+
SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
SelectionDAG &DAG) const {
assert(Op.getValueType().isScalableVector() &&
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index b68e06dede580..e1ac77334d2c7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -765,6 +765,7 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerEXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerVECTOR_REPEAT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index e9191c95eee35..4b6a89d863d89 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -1587,6 +1587,14 @@ multiclass sve_int_perm_dup_i<string asm> {
(!cast<Instruction>(NAME # _D) (SUBREG_TO_REG $vec, dsub), sve_elm_idx_extdup_d:$index)>;
}
+ // Broadcast whole fixed-length vectors to packed SVE types.
+ // See LowerVECTOR_REPEAT for handling of legal unpacked types.
+ foreach VT = [nxv16i8, nxv8i16, nxv8f16, nxv8bf16,
+ nxv4i32, nxv4f32, nxv2i64, nxv2f64] in {
+ def : Pat<(VT (vector_repeat (SVEType<VT>.ZSub V128:$vec))),
+ (!cast<Instruction>(NAME # _Q) (SUBREG_TO_REG $vec, zsub), (i64 0))>;
+ }
+
// When extracting from an unpacked vector the index must be scaled to account
// for the "holes" in the underlying packed vector type. We get the scaling
// for free by "promoting" the element type to one whose underlying vector
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-repeat.ll b/llvm/test/CodeGen/AArch64/sve-vector-repeat.ll
new file mode 100644
index 0000000000000..5d2f550e9b2f4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vector-repeat.ll
@@ -0,0 +1,499 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+
+define <vscale x 16 x i8> @repeat_quad_i8(<16 x i8> %a) {
+; CHECK-LABEL: repeat_quad_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 16 x i8> @llvm.vector.repeat.nxv16i8.v16i8(<16 x i8> %a)
+ ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 16 x i8> @repeat_double_i8(<8 x i8> %a) {
+; CHECK-LABEL: repeat_double_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %tmp = shufflevector <8 x i8> %a, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %out = call <vscale x 16 x i8> @llvm.vector.repeat.nxv16i8.v16i8(<16 x i8> %tmp)
+ ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @repeat_double_i8_to_double_sve(<8 x i8> %a) {
+; CHECK-LABEL: repeat_double_i8_to_double_sve:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: sxtb z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i8> @llvm.vector.repeat.nxv8i8.v8i8(<8 x i8> %a)
+ %out.legal = sext <vscale x 8 x i8> %out to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %out.legal
+}
+
+define <vscale x 16 x i8> @repeat_one_i8(<8 x i8> %a.legal) {
+; CHECK-LABEL: repeat_one_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.b, b0
+; CHECK-NEXT: ret
+ %a = call <1 x i8> @llvm.vector.extract.v1i8.nxv1i8(<8 x i8> %a.legal, i64 0)
+ %out = call <vscale x 1 x i8> @llvm.vector.repeat.nxv1i8.v1i8(<1 x i8> %a)
+ %out.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(<vscale x 16 x i8> poison, <vscale x 1 x i8> %out, i64 0)
+ ret <vscale x 16 x i8> %out.legal
+}
+
+define <vscale x 8 x i16> @repeat_quad_i16(<8 x i16> %a) {
+; CHECK-LABEL: repeat_quad_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i16> @llvm.vector.repeat.nxv8i16.v8i16(<8 x i16> %a)
+ ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 16 x i8> @repeat_wide_i16(<8 x i16> %a.lo, <8 x i16> %a.hi) {
+; CHECK-LABEL: repeat_wide_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp2 v2.8h, v0.8h, v1.8h
+; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: mov z1.q, q2
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: zip2 z2.h, z0.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.b, z0.b, z2.b
+; CHECK-NEXT: ret
+ %a = shufflevector <8 x i16> %a.lo, <8 x i16> %a.hi,
+ <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %out = call <vscale x 16 x i16> @llvm.vector.repeat.nxv16i16.v16i16(<16 x i16> %a)
+ %out.legal = trunc <vscale x 16 x i16> %out to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %out.legal
+}
+
+define <vscale x 8 x i16> @repeat_double_i16(<4 x i16> %a) {
+; CHECK-LABEL: repeat_double_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %tmp = shufflevector <4 x i16> %a, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %out = call <vscale x 8 x i16> @llvm.vector.repeat.nxv8i16.v8i16(<8 x i16> %tmp)
+ ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 8 x i16> @repeat_one_i16(<4 x i16> %a.legal) {
+; CHECK-LABEL: repeat_one_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.h, h0
+; CHECK-NEXT: ret
+ %a = call <1 x i16> @llvm.vector.extract.v1i16.nxv1i16(<4 x i16> %a.legal, i64 0)
+ %out = call <vscale x 1 x i16> @llvm.vector.repeat.nxv1i16.v1i16(<1 x i16> %a)
+ %out.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(<vscale x 8 x i16> poison, <vscale x 1 x i16> %out, i64 0)
+ ret <vscale x 8 x i16> %out.legal
+}
+
+define <vscale x 4 x i32> @repeat_double_i16_to_double_sve(<4 x i16> %a) {
+; CHECK-LABEL: repeat_double_i16_to_double_sve:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i16> @llvm.vector.repeat.nxv4i16.v4i16(<4 x i16> %a)
+ %out.legal = zext <vscale x 4 x i16> %out to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %out.legal
+}
+
+define <vscale x 4 x i32> @repeat_quad_i32(<4 x i32> %a) {
+; CHECK-LABEL: repeat_quad_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.vector.repeat.nxv4i32.v4i32(<4 x i32> %a)
+ ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 4 x i32> @repeat_double_i32(<2 x i32> %a) {
+; CHECK-LABEL: repeat_double_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %tmp = shufflevector <2 x i32> %a, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %out = call <vscale x 4 x i32> @llvm.vector.repeat.nxv4i32.v4i32(<4 x i32> %tmp)
+ ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 4 x i32> @repeat_one_i32(<2 x i32> %a.legal) {
+; CHECK-LABEL: repeat_one_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.s, s0
+; CHECK-NEXT: ret
+ %a = call <1 x i32> @llvm.vector.extract.v1i32.nxv1i32(<2 x i32> %a.legal, i64 0)
+ %out = call <vscale x 1 x i32> @llvm.vector.repeat.nxv1i32.v1i32(<1 x i32> %a)
+ %out.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> poison, <vscale x 1 x i32> %out, i64 0)
+ ret <vscale x 4 x i32> %out.legal
+}
+
+define <vscale x 2 x i64> @repeat_quad_i64(<2 x i64> %a) {
+; CHECK-LABEL: repeat_quad_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i64> @llvm.vector.repeat.nxv2i64.v2i64(<2 x i64> %a)
+ ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i64> @repeat_double_i64(<1 x i64> %a) {
+; CHECK-LABEL: repeat_double_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: dup v0.2d, v0.d[0]
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %tmp = shufflevector <1 x i64> %a, <1 x i64> poison, <2 x i32> zeroinitializer
+ %out = call <vscale x 2 x i64> @llvm.vector.repeat.nxv2i64.v2i64(<2 x i64> %tmp)
+ ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i64> @repeat_one_i64(<1 x i64> %a) {
+; CHECK-LABEL: repeat_one_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: ret
+ %out = call <vscale x 1 x i64> @llvm.vector.repeat.nxv1i64.v1i64(<1 x i64> %a)
+ %out.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %out, i64 0)
+ ret <vscale x 2 x i64> %out.legal
+}
+
+; vscale_range tests
+
+; wider-than-NEON fixed-length source and wide destination
+define <vscale x 4 x i32> @repeat_v4i64_to_nxv4i64(<vscale x 2 x i64> %a.legal) vscale_range(2,8) {
+; CHECK-LABEL: repeat_v4i64_to_nxv4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT: uzp2 v2.2d, v0.2d, v1.2d
+; CHECK-NEXT: uzp1 v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: mov z1.q, q2
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: zip2 z2.d, z0.d, z1.d
+; CHECK-NEXT: zip1 z0.d, z0.d, z1.d
+; CHECK-NEXT: uzp1 z0.s, z0.s, z2.s
+; CHECK-NEXT: ret
+ %a = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> %a.legal, i64 0)
+ %r = call <vscale x 4 x i64> @llvm.vector.repeat.nxv4i64.v4i64(<4 x i64> %a)
+ %r.legal = trunc <vscale x 4 x i64> %r to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %r.legal
+}
+
+; FP / BFP types
+
+define <vscale x 8 x half> @repeat_quad_f16(<8 x half> %a) {
+; CHECK-LABEL: repeat_quad_f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x half> @llvm.vector.repeat.nxv8f16(<8 x half> %a)
+ ret <vscale x 8 x half> %out
+}
+
+define <vscale x 8 x half> @repeat_double_f16(<4 x half> %a) {
+; CHECK-LABEL: repeat_double_f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %tmp = shufflevector <4 x half> %a, <4 x half> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %out = call <vscale x 8 x half> @llvm.vector.repeat.nxv8f16(<8 x half> %tmp)
+ ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x half> @repeat_double_f16_to_double_sve(<4 x half> %a) {
+; CHECK-LABEL: repeat_double_f16_to_double_sve:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x half> @llvm.vector.repeat.nxv4f16(<4 x half> %a)
+ ret <vscale x 4 x half> %out
+}
+
+define <vscale x 2 x half> @repeat_2f16_to_nxv2f16(<4 x half> %a) {
+; CHECK-LABEL: repeat_2f16_to_nxv2f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: dup v0.2s, v0.s[0]
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+ %a.legal = call <2 x half> @llvm.vector.extract.v2f16.v4f16(<4 x half> %a, i64 0)
+ %out = call <vscale x 2 x half> @llvm.vector.repeat.nxv2f16(<2 x half> %a.legal)
+ ret <vscale x 2 x half> %out
+}
+
+define <vscale x 8 x half> @repeat_one_f16(<4 x half> %a.legal) {
+; CHECK-LABEL: repeat_one_f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.h, h0
+; CHECK-NEXT: ret
+ %a = call <1 x half> @llvm.vector.extract.v1f16.v4f16(<4 x half> %a.legal, i64 0)
+ %out = call <vscale x 1 x half> @llvm.vector.repeat.nxv1f16.v1f16(<1 x half> %a)
+ %out.legal = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.nxv1f16(<vscale x 8 x half> poison, <vscale x 1 x half> %out, i64 0)
+ ret <vscale x 8 x half> %out.legal
+}
+
+define <vscale x 8 x bfloat> @repeat_quad_bf16(<8 x bfloat> %a) #0 {
+; CHECK-LABEL: repeat_quad_bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x bfloat> @llvm.vector.repeat.nxv8bf16.v8bf16(<8 x bfloat> %a)
+ ret <vscale x 8 x bfloat> %out
+}
+
+define <vscale x 8 x bfloat> @repeat_one_bf16(<4 x bfloat> %a.legal) #0 {
+; CHECK-LABEL: repeat_one_bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.h, h0
+; CHECK-NEXT: ret
+ %a = call <1 x bfloat> @llvm.vector.extract.v1bf16.v4bf16(<4 x bfloat> %a.legal, i64 0)
+ %out = call <vscale x 1 x bfloat> @llvm.vector.repeat.nxv1bf16.v1bf16(<1 x bfloat> %a)
+ %out.legal = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.nxv1bf16(<vscale x 8 x bfloat> poison, <vscale x 1 x bfloat> %out, i64 0)
+ ret <vscale x 8 x bfloat> %out.legal
+}
+
+define <vscale x 4 x float> @repeat_quad_f32(<4 x float> %a) {
+; CHECK-LABEL: repeat_quad_f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x float> @llvm.vector.repeat.nxv4f32.v4f32(<4 x float> %a)
+ ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x float> @repeat_double_f32_to_nxv2f32(<2 x float> %a) {
+; CHECK-LABEL: repeat_double_f32_to_nxv2f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x float> @llvm.vector.repeat.nxv2f32.v2f32(<2 x float> %a)
+ ret <vscale x 2 x float> %out
+}
+
+define <vscale x 4 x float> @repeat_one_f32(<2 x float> %a.legal) {
+; CHECK-LABEL: repeat_one_f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.s, s0
+; CHECK-NEXT: ret
+ %a = call <1 x float> @llvm.vector.extract.v1f32.v2f32(<2 x float> %a.legal, i64 0)
+ %out = call <vscale x 1 x float> @llvm.vector.repeat.nxv1f32.v1f32(<1 x float> %a)
+ %out.legal = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv1f32(<vscale x 4 x float> poison, <vscale x 1 x float> %out, i64 0)
+ ret <vscale x 4 x float> %out.legal
+}
+
+define <vscale x 4 x bfloat> @repeat_double_bf16_to_nxv4bf16(<4 x bfloat> %a) #0 {
+; CHECK-LABEL: repeat_double_bf16_to_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x bfloat> @llvm.vector.repeat.nxv4bf16.v4bf16(<4 x bfloat> %a)
+ ret <vscale x 4 x bfloat> %out
+}
+
+define <vscale x 2 x double> @repeat_quad_f64(<2 x double> %a) {
+; CHECK-LABEL: repeat_quad_f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x double> @llvm.vector.repeat.nxv2f64.v2f64(<2 x double> %a)
+ ret <vscale x 2 x double> %out
+}
+
+define <vscale x 2 x double> @repeat_one_f64(<1 x double> %a) {
+; CHECK-LABEL: repeat_one_f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: ret
+ %out = call <vscale x 1 x double> @llvm.vector.repeat.nxv1f64.v1f64(<1 x double> %a)
+ %out.legal = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.nxv1f64(<vscale x 2 x double> poison, <vscale x 1 x double> %out, i64 0)
+ ret <vscale x 2 x double> %out.legal
+}
+
+; Predicates
+
+define <vscale x 16 x i1> @repeat_v16i1_to_nxv16i1(<16 x i8> %a) {
+; CHECK-LABEL: repeat_v16i1_to_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.b, z0.b, #0x1
+; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <16 x i8> %a to <16 x i1>
+ %out = call <vscale x 16 x i1> @llvm.vector.repeat.nxv16i1.v16i1(<16 x i1> %a.legal)
+ ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @repeat_v8i1_to_nxv16i1(<8 x i8> %a) {
+; CHECK-LABEL: repeat_v8i1_to_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.b, z0.b, #0x1
+; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <8 x i8> %a to <8 x i1>
+ %tmp = shufflevector <8 x i1> %a.legal, <8 x i1> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %out = call <vscale x 16 x i1> @llvm.vector.repeat.nxv16i1.v16i1(<16 x i1> %tmp)
+ ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @repeat_v8i1_double_to_nxv8i1(<8 x i8> %a) {
+; CHECK-LABEL: repeat_v8i1_double_to_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.h, z0.h, #0x1
+; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <8 x i8> %a to <8 x i1>
+ %out = call <vscale x 8 x i1> @llvm.vector.repeat.nxv8i1.v8i1(<8 x i1> %a.legal)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @repeat_v8i1_quad_to_nxv8i1(<8 x i16> %a) {
+; CHECK-LABEL: repeat_v8i1_quad_to_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.h, z0.h, #0x1
+; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <8 x i16> %a to <8 x i1>
+ %out = call <vscale x 8 x i1> @llvm.vector.repeat.nxv8i1.v8i1(<8 x i1> %a.legal)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @repeat_v4i1_double_to_nxv8i1(<4 x i16> %a) {
+; CHECK-LABEL: repeat_v4i1_double_to_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.h, z0.h, #0x1
+; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <4 x i16> %a to <4 x i1>
+ %tmp = shufflevector <4 x i1> %a.legal, <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %out = call <vscale x 8 x i1> @llvm.vector.repeat.nxv8i1.v8i1(<8 x i1> %tmp)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @repeat_v4i1_double_to_nxv4i1(<4 x i16> %a) {
+; CHECK-LABEL: repeat_v4i1_double_to_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.s, z0.s, #0x1
+; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <4 x i16> %a to <4 x i1>
+ %out = call <vscale x 4 x i1> @llvm.vector.repeat.nxv4i1.v4i1(<4 x i1> %a.legal)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @repeat_v4i1_quad_to_nxv4i1(<4 x i32> %a) {
+; CHECK-LABEL: repeat_v4i1_quad_to_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.s, z0.s, #0x1
+; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <4 x i32> %a to <4 x i1>
+ %out = call <vscale x 4 x i1> @llvm.vector.repeat.nxv4i1.v4i1(<4 x i1> %a.legal)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @repeat_v2i1_double_to_nxv4i1(<2 x i32> %a) {
+; CHECK-LABEL: repeat_v2i1_double_to_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.s, z0.s, #0x1
+; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <2 x i32> %a to <2 x i1>
+ %tmp = shufflevector <2 x i1> %a.legal, <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %out = call <vscale x 4 x i1> @llvm.vector.repeat.nxv4i1.v4i1(<4 x i1> %tmp)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @repeat_v2i1_double_to_nxv2i1(<2 x i32> %a) {
+; CHECK-LABEL: repeat_v2i1_double_to_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.d, z0.d, #0x1
+; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <2 x i32> %a to <2 x i1>
+ %out = call <vscale x 2 x i1> @llvm.vector.repeat.nxv2i1.v2i1(<2 x i1> %a.legal)
+ ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @repeat_v2i1_quad_to_nxv2i1(<2 x i64> %a) {
+; CHECK-LABEL: repeat_v2i1_quad_to_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.d, z0.d, #0x1
+; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <2 x i64> %a to <2 x i1>
+ %out = call <vscale x 2 x i1> @llvm.vector.repeat.nxv2i1.v2i1(<2 x i1> %a.legal)
+ ret <vscale x 2 x i1> %out
+}
diff --git a/llvm/test/Verifier/vector-repeat-intrinsic-invalid.ll b/llvm/test/Verifier/vector-repeat-intrinsic-invalid.ll
new file mode 100644
index 0000000000000..da2c261556cbb
--- /dev/null
+++ b/llvm/test/Verifier/vector-repeat-intrinsic-invalid.ll
@@ -0,0 +1,25 @@
+; RUN: not opt -passes=verify -disable-output < %s 2>&1 | FileCheck %s
+
+; CHECK: vector_repeat argument and result must have the same element type.
+define <vscale x 8 x i32> @mismatched_element_types(<8 x i64> %vec) {
+ %result = call <vscale x 8 x i32> @llvm.vector.repeat.nxv8i32.v8i64(<8 x i64> %vec)
+ ret <vscale x 8 x i32> %result
+}
+
+; CHECK: vector_repeat result must be a scalable vector.
+define <8 x i32> @fixed_to_fixed(<8 x i32> %vec) {
+ %result = call <8 x i32> @llvm.vector.repeat.v8i32(<8 x i32> %vec)
+ ret <8 x i32> %result
+}
+
+; CHECK: vector_repeat argument must be a fixed-length vector.
+define <vscale x 8 x i32> @scalable_to_scalable(<vscale x 8 x i32> %vec) {
+ %result = call <vscale x 8 x i32> @llvm.vector.repeat.nxv8i32(<vscale x 8 x i32> %vec)
+ ret <vscale x 8 x i32> %result
+}
+
+; CHECK: vector_repeat argument and result must have the same minimum element count.
+define <vscale x 8 x i32> @mismatched_minimum_element_count(<4 x i32> %vec) {
+ %result = call <vscale x 8 x i32> @llvm.vector.repeat.nxv8i32.v4i32(<4 x i32> %vec)
+ ret <vscale x 8 x i32> %result
+}
diff --git a/llvm/test/Verifier/vector-repeat-intrinsic-valid.ll b/llvm/test/Verifier/vector-repeat-intrinsic-valid.ll
new file mode 100644
index 0000000000000..a6bf98e38d7b4
--- /dev/null
+++ b/llvm/test/Verifier/vector-repeat-intrinsic-valid.ll
@@ -0,0 +1,8 @@
+; RUN: opt -passes=verify -disable-output < %s
+
+; Test that a correctly formed vector.repeat passes verifier checks.
+
+define <vscale x 4 x i32> @fixed_to_scalable(<4 x i32> %vec) {
+ %result = call <vscale x 4 x i32> @llvm.vector.repeat.nxv4i32.v4i32(<4 x i32> %vec)
+ ret <vscale x 4 x i32> %result
+}
diff --git a/llvm/unittests/CodeGen/SelectionDAGNodeConstructionTest.cpp b/llvm/unittests/CodeGen/SelectionDAGNodeConstructionTest.cpp
index c2c1b52926c5a..2f5ae1ac52c1c 100644
--- a/llvm/unittests/CodeGen/SelectionDAGNodeConstructionTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGNodeConstructionTest.cpp
@@ -525,3 +525,16 @@ TEST_F(SelectionDAGNodeConstructionTest, ExpandPartialReduceSUMLA) {
EXPECT_EQ(NumSignExtends, 1u);
EXPECT_EQ(NumZeroExtends, 1u);
}
+
+// Verify that a nxv1 vector_repeat gets canonicalised as splat_vector
+TEST_F(SelectionDAGNodeConstructionTest, VectorRepeat) {
+ SDLoc DL;
+ SDValue V1 = DAG->getCopyFromReg(DAG->getEntryNode(), DL,
+ Register::index2VirtReg(1), MVT::v1i32);
+ SDValue NXV1 = DAG->getNode(ISD::VECTOR_REPEAT, DL, MVT::nxv1i32, V1);
+
+ ASSERT_EQ(NXV1.getOpcode(), ISD::SPLAT_VECTOR);
+ ASSERT_EQ(NXV1.getOperand(0).getOpcode(), ISD::EXTRACT_VECTOR_ELT);
+ EXPECT_EQ(NXV1.getOperand(0).getOperand(0), V1);
+ EXPECT_TRUE(isNullConstant(NXV1.getOperand(0).getOperand(1)));
+}
More information about the llvm-commits
mailing list