[llvm] [X86][GlobalIsel] Support G_BUILD_VECTOR for non constant using G_INSERT_VECTOR_ELT (PR #204078)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 01:20:57 PDT 2026
https://github.com/mahesh-attarde updated https://github.com/llvm/llvm-project/pull/204078
>From 826b22a27dd9b20ef93c4a298492ca96382a32f9 Mon Sep 17 00:00:00 2001
From: mattarde <mattarde at intel.com>
Date: Tue, 16 Jun 2026 01:02:37 -0700
Subject: [PATCH 1/2] [X86][GlobalIsel] Support G_BUILD_VECTOR for non constant
using G_INSERT_VECTOR_ELT
---
.../X86/GISel/X86InstructionSelector.cpp | 108 +++
.../lib/Target/X86/GISel/X86LegalizerInfo.cpp | 109 ++-
llvm/lib/Target/X86/GISel/X86LegalizerInfo.h | 3 +
.../CodeGen/X86/GlobalISel/build-vector.ll | 885 ++++++++++++++++++
4 files changed, 1089 insertions(+), 16 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/GlobalISel/build-vector.ll
diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
index 3c94c33cdaaa6..ae8ad6823acb1 100644
--- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
+++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
@@ -121,6 +121,11 @@ class X86InstructionSelector : public InstructionSelector {
MachineFunction &MF) const;
bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
MachineFunction &MF) const;
+ bool selectInsertVectorElt(MachineInstr &I, MachineRegisterInfo &MRI,
+ MachineFunction &MF);
+
+ unsigned getMachineOpcodeForInsert(unsigned EltSizeInBits) const;
+ unsigned getScalarToVecOpcode(unsigned EltSizeInBits) const;
ComplexRendererFns selectAddr(MachineOperand &Root) const;
@@ -491,6 +496,8 @@ bool X86InstructionSelector::select(MachineInstr &I) {
return selectMulDivRem(I, MRI, MF);
case TargetOpcode::G_SELECT:
return selectSelect(I, MRI, MF);
+ case TargetOpcode::G_INSERT_VECTOR_ELT:
+ return selectInsertVectorElt(I, MRI, MF);
}
return false;
@@ -1971,6 +1978,107 @@ bool X86InstructionSelector::selectSelect(MachineInstr &I,
return true;
}
+unsigned X86InstructionSelector::getMachineOpcodeForInsert(
+ unsigned EltSizeInBits) const {
+ bool HasAVX = STI.hasAVX();
+ bool HasAVX512 = STI.hasAVX512();
+ bool HasBWI = STI.hasBWI();
+ bool HasDQI = STI.hasDQI();
+ bool HasVLX = STI.hasVLX();
+ bool HasSSE41 = STI.hasSSE41();
+
+ // {SSE, AVX, AVX512} opcodes per element size.
+ static constexpr unsigned PinsrOpcodes[][3] = {
+ {X86::PINSRBrri, X86::VPINSRBrri, X86::VPINSRBZrri}, // 8-bit
+ {X86::PINSRWrri, X86::VPINSRWrri, X86::VPINSRWZrri}, // 16-bit
+ {X86::PINSRDrri, X86::VPINSRDrri, X86::VPINSRDZrri}, // 32-bit
+ {X86::PINSRQrri, X86::VPINSRQrri, X86::VPINSRQZrri}, // 64-bit
+ };
+ assert(EltSizeInBits % 8 == 0 && EltSizeInBits <= 64);
+ unsigned Row = EltSizeInBits / 8 - 1;
+ if (EltSizeInBits == 8) {
+ if (HasBWI)
+ return PinsrOpcodes[Row][2];
+ if (HasAVX)
+ return PinsrOpcodes[Row][1];
+ if (HasSSE41)
+ return PinsrOpcodes[Row][0];
+ return 0;
+ }
+ if (EltSizeInBits == 16) {
+ if (HasBWI)
+ return PinsrOpcodes[Row][2];
+ if (HasAVX)
+ return PinsrOpcodes[Row][1];
+ return PinsrOpcodes[Row][0];
+ }
+ // 32/64-bit
+ if (HasAVX512 && (HasDQI || HasVLX))
+ return PinsrOpcodes[Row][2];
+ if (HasAVX)
+ return PinsrOpcodes[Row][1];
+ if (HasSSE41)
+ return PinsrOpcodes[Row][0];
+ return 0;
+}
+
+unsigned
+X86InstructionSelector::getScalarToVecOpcode(unsigned EltSizeInBits) const {
+ bool HasAVX = STI.hasAVX();
+ bool HasAVX512 = STI.hasAVX512();
+ switch (EltSizeInBits) {
+ case 32:
+ return HasAVX512 ? X86::VMOVDI2PDIZrr
+ : HasAVX ? X86::VMOVDI2PDIrr
+ : X86::MOVDI2PDIrr;
+ case 64:
+ return HasAVX512 ? X86::VMOV64toPQIZrr
+ : HasAVX ? X86::VMOV64toPQIrr
+ : X86::MOV64toPQIrr;
+ default:
+ llvm_unreachable("Unsupported element size");
+ }
+ return 0;
+}
+
+bool X86InstructionSelector::selectInsertVectorElt(MachineInstr &I,
+ MachineRegisterInfo &MRI,
+ MachineFunction &MF) {
+ assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
+
+ Register DstReg = I.getOperand(0).getReg();
+ Register VecReg = I.getOperand(1).getReg();
+ Register EltReg = I.getOperand(2).getReg();
+ Register IdxReg = I.getOperand(3).getReg();
+
+ LLT VecTy = MRI.getType(DstReg);
+ unsigned EltSize = VecTy.getScalarSizeInBits();
+ unsigned NumElts = VecTy.getNumElements();
+ unsigned VecSize = VecTy.getSizeInBits();
+
+ auto IdxVal = getIConstantVRegValWithLookThrough(IdxReg, MRI);
+ if (!IdxVal)
+ return false;
+ uint64_t Idx = IdxVal->Value.getZExtValue();
+ if (Idx >= NumElts)
+ return false;
+
+ // TODO: use movd/vmovd/movq at idx 0 to avoid false dependency.
+
+ unsigned Opc = getMachineOpcodeForInsert(EltSize);
+ if (!Opc)
+ return false;
+
+ auto &MIB = *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opc), DstReg)
+ .addReg(VecReg)
+ .addReg(EltReg)
+ .addImm(Idx);
+
+ constrainSelectedInstRegOperands(MIB, TII, TRI, RBI);
+ I.eraseFromParent();
+ return true;
+}
+
InstructionSelector::ComplexRendererFns
X86InstructionSelector::selectAddr(MachineOperand &Root) const {
MachineInstr *MI = Root.getParent();
diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
index 750bb03df21c7..7fe090f4ef131 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
@@ -560,6 +560,24 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
.clampNumElements(0, v2s64, s64MaxVector)
.moreElementsToNextPow2(0);
+ getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
+ .legalFor(HasSSE1, {{v4s32, s32, sMaxScalar}})
+ .legalFor(HasSSE2, {{v16s8, s8, sMaxScalar},
+ {v8s16, s16, sMaxScalar},
+ {v2s64, s64, sMaxScalar}})
+ .customIf([=](const LegalityQuery &Query) {
+ const LLT VecTy = Query.Types[0];
+ const LLT EltTy = Query.Types[1];
+ if (EltTy != VecTy.getElementType())
+ return false;
+ if (VecTy.getSizeInBits() <= 128)
+ return false;
+ return (HasAVX && (VecTy == v32s8 || VecTy == v16s16 ||
+ VecTy == v8s32 || VecTy == v4s64)) ||
+ (HasAVX512 && (VecTy == v64s8 || VecTy == v32s16 ||
+ VecTy == v16s32 || VecTy == v8s64));
+ });
+
getActionDefinitionsBuilder({G_EXTRACT, G_INSERT})
.legalIf([=](const LegalityQuery &Query) {
unsigned SubIdx = Query.Opcode == G_EXTRACT ? 0 : 1;
@@ -638,6 +656,8 @@ bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
return false;
case TargetOpcode::G_BUILD_VECTOR:
return legalizeBuildVector(MI, MRI, Helper);
+ case TargetOpcode::G_INSERT_VECTOR_ELT:
+ return legalizeInsertVectorElt(MI, MRI, Helper);
case TargetOpcode::G_FPTOUI:
return legalizeFPTOUI(MI, MRI, Helper);
case TargetOpcode::G_UITOFP:
@@ -723,10 +743,13 @@ bool X86LegalizerInfo::legalizeBuildVector(MachineInstr &MI,
LLT DstTy = MRI.getType(Dst);
MachineFunction &MF = MIRBuilder.getMF();
LLVMContext &Ctx = MF.getFunction().getContext();
- uint64_t DstTySize = DstTy.getScalarSizeInBits();
+ unsigned EltSize = DstTy.getScalarSizeInBits();
+ unsigned NumElts = BuildVector.getNumSources();
- SmallVector<Constant *, 4> CstIdxs;
- for (unsigned i = 0; i < BuildVector.getNumSources(); ++i) {
+ // Try constant-pool materialization for all-constant vectors.
+ bool AllConst = true;
+ SmallVector<Constant *, 16> CstIdxs;
+ for (unsigned i = 0; i < NumElts; ++i) {
Register Source = BuildVector.getSourceReg(i);
auto ValueAndReg = getIConstantVRegValWithLookThrough(Source, MRI);
@@ -742,25 +765,79 @@ bool X86LegalizerInfo::legalizeBuildVector(MachineInstr &MI,
}
if (getOpcodeDef<GImplicitDef>(Source, MRI)) {
- CstIdxs.emplace_back(UndefValue::get(Type::getIntNTy(Ctx, DstTySize)));
+ CstIdxs.emplace_back(UndefValue::get(Type::getIntNTy(Ctx, EltSize)));
continue;
}
- return false;
+ AllConst = false;
+ break;
}
- Constant *ConstVal = ConstantVector::get(CstIdxs);
+ if (AllConst) {
+ Constant *ConstVal = ConstantVector::get(CstIdxs);
+ const DataLayout &DL = MIRBuilder.getDataLayout();
+ unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
+ Align Alignment(DL.getABITypeAlign(ConstVal->getType()));
+ auto Addr = MIRBuilder.buildConstantPool(
+ LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)),
+ MF.getConstantPool()->getConstantPoolIndex(ConstVal, Alignment));
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),
+ MachineMemOperand::MOLoad, DstTy, Alignment);
+ MIRBuilder.buildLoad(Dst, Addr, *MMO);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // Non-constant: lower to a sequence of G_INSERT_VECTOR_ELT.
+ LLT IdxTy = LLT::scalar(Subtarget.is64Bit() ? 64 : 32);
+
+ Register Vec = MIRBuilder.buildUndef(DstTy).getReg(0);
+ for (unsigned i = 0; i < NumElts; ++i) {
+ Register Src = BuildVector.getSourceReg(i);
+ if (getOpcodeDef<GImplicitDef>(Src, MRI))
+ continue;
+ auto Idx = MIRBuilder.buildConstant(IdxTy, i);
+ Vec = MIRBuilder.buildInsertVectorElement(DstTy, Vec, Src, Idx).getReg(0);
+ }
+
+ MIRBuilder.buildCopy(Dst, Vec);
+ MI.eraseFromParent();
+ return true;
+}
+
+bool X86LegalizerInfo::legalizeInsertVectorElt(MachineInstr &MI,
+ MachineRegisterInfo &MRI,
+ LegalizerHelper &Helper) const {
+ MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
+ Register Dst = MI.getOperand(0).getReg();
+ Register Vec = MI.getOperand(1).getReg();
+ Register Elt = MI.getOperand(2).getReg();
+ Register IdxReg = MI.getOperand(3).getReg();
+
+ LLT VecTy = MRI.getType(Dst);
+ unsigned EltSize = VecTy.getScalarSizeInBits();
+ unsigned SubNumElts = 128 / EltSize;
+ LLT SubVecTy = LLT::fixed_vector(SubNumElts, EltSize);
+ LLT IdxTy = LLT::scalar(Subtarget.is64Bit() ? 64 : 32);
+
+ auto IdxVal = getIConstantVRegValWithLookThrough(IdxReg, MRI);
+ if (!IdxVal)
+ return false;
+ uint64_t Idx = IdxVal->Value.getZExtValue();
+ unsigned SubVecIdx = Idx / SubNumElts;
+ unsigned EltIdx = Idx % SubNumElts;
+
+ // Extract 128-bit subvector.
+ auto SubVec = MIRBuilder.buildExtract(SubVecTy, Vec, SubVecIdx * 128);
+
+ // Insert element into the 128-bit subvector.
+ auto NewIdx = MIRBuilder.buildConstant(IdxTy, EltIdx);
+ auto NewSubVec =
+ MIRBuilder.buildInsertVectorElement(SubVecTy, SubVec, Elt, NewIdx);
- const DataLayout &DL = MIRBuilder.getDataLayout();
- unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
- Align Alignment(DL.getABITypeAlign(ConstVal->getType()));
- auto Addr = MIRBuilder.buildConstantPool(
- LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)),
- MF.getConstantPool()->getConstantPoolIndex(ConstVal, Alignment));
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),
- MachineMemOperand::MOLoad, DstTy, Alignment);
+ // Re-insert 128-bit subvector into the wide vector.
+ MIRBuilder.buildInsert(Dst, Vec, NewSubVec, SubVecIdx * 128);
- MIRBuilder.buildLoad(Dst, Addr, *MMO);
MI.eraseFromParent();
return true;
}
diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
index 58be7bb7d02b8..2300ce1ddf1ea 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
@@ -40,6 +40,9 @@ class X86LegalizerInfo : public LegalizerInfo {
bool legalizeBuildVector(MachineInstr &MI, MachineRegisterInfo &MRI,
LegalizerHelper &Helper) const;
+ bool legalizeInsertVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI,
+ LegalizerHelper &Helper) const;
+
bool legalizeFPTOUI(MachineInstr &MI, MachineRegisterInfo &MRI,
LegalizerHelper &Helper) const;
diff --git a/llvm/test/CodeGen/X86/GlobalISel/build-vector.ll b/llvm/test/CodeGen/X86/GlobalISel/build-vector.ll
new file mode 100644
index 0000000000000..fc103ff64995f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/build-vector.ll
@@ -0,0 +1,885 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc -mtriple=x86_64-linux-gnu -mattr=+sse4.1 -global-isel -global-isel-abort=1 < %s | FileCheck %s --check-prefixes=GISEL-SSE41
+; RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx -global-isel -global-isel-abort=1 < %s | FileCheck %s --check-prefixes=GISEL-AVX
+; RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f,+avx512bw,+avx512dq -global-isel -global-isel-abort=1 < %s | FileCheck %s --check-prefixes=GISEL-AVX
+; RUN: llc -mtriple=x86_64-linux-gnu -mattr=+sse4.1 < %s | FileCheck %s --check-prefixes=SDAG-SSE41
+; RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx < %s | FileCheck %s --check-prefixes=SDAG-AVX
+; RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f,+avx512bw,+avx512dq < %s | FileCheck %s --check-prefixes=SDAG-AVX
+
+;128-bit integer vectors
+
+define <4 x i32> @build_vector_v4i32(i32 %a, i32 %b, i32 %c, i32 %d) {
+; GISEL-SSE41-LABEL: build_vector_v4i32:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: pinsrd $0, %edi, %xmm0
+; GISEL-SSE41-NEXT: pinsrd $1, %esi, %xmm0
+; GISEL-SSE41-NEXT: pinsrd $2, %edx, %xmm0
+; GISEL-SSE41-NEXT: pinsrd $3, %ecx, %xmm0
+; GISEL-SSE41-NEXT: retq
+;
+; GISEL-AVX-LABEL: build_vector_v4i32:
+; GISEL-AVX: # %bb.0:
+; GISEL-AVX-NEXT: vpinsrd $0, %edi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; GISEL-AVX-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v4i32:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: movd %edi, %xmm0
+; SDAG-SSE41-NEXT: pinsrd $1, %esi, %xmm0
+; SDAG-SSE41-NEXT: pinsrd $2, %edx, %xmm0
+; SDAG-SSE41-NEXT: pinsrd $3, %ecx, %xmm0
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v4i32:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vmovd %edi, %xmm0
+; SDAG-AVX-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; SDAG-AVX-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SDAG-AVX-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SDAG-AVX-NEXT: retq
+ %v0 = insertelement <4 x i32> undef, i32 %a, i64 0
+ %v1 = insertelement <4 x i32> %v0, i32 %b, i64 1
+ %v2 = insertelement <4 x i32> %v1, i32 %c, i64 2
+ %v3 = insertelement <4 x i32> %v2, i32 %d, i64 3
+ ret <4 x i32> %v3
+}
+
+define <2 x i64> @build_vector_v2i64(i64 %a, i64 %b) {
+; GISEL-SSE41-LABEL: build_vector_v2i64:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: pinsrq $0, %rdi, %xmm0
+; GISEL-SSE41-NEXT: pinsrq $1, %rsi, %xmm0
+; GISEL-SSE41-NEXT: retq
+;
+; GISEL-AVX-LABEL: build_vector_v2i64:
+; GISEL-AVX: # %bb.0:
+; GISEL-AVX-NEXT: vpinsrq $0, %rdi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vpinsrq $1, %rsi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v2i64:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: movq %rsi, %xmm1
+; SDAG-SSE41-NEXT: movq %rdi, %xmm0
+; SDAG-SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v2i64:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vmovq %rsi, %xmm0
+; SDAG-AVX-NEXT: vmovq %rdi, %xmm1
+; SDAG-AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SDAG-AVX-NEXT: retq
+ %v0 = insertelement <2 x i64> undef, i64 %a, i64 0
+ %v1 = insertelement <2 x i64> %v0, i64 %b, i64 1
+ ret <2 x i64> %v1
+}
+
+;128-bit floating-point vectors
+
+define <4 x float> @build_vector_v4f32(float %a, float %b, float %c, float %d) {
+; GISEL-SSE41-LABEL: build_vector_v4f32:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: movd %xmm0, %eax
+; GISEL-SSE41-NEXT: pinsrd $0, %eax, %xmm0
+; GISEL-SSE41-NEXT: movd %xmm1, %eax
+; GISEL-SSE41-NEXT: pinsrd $1, %eax, %xmm0
+; GISEL-SSE41-NEXT: movd %xmm2, %eax
+; GISEL-SSE41-NEXT: pinsrd $2, %eax, %xmm0
+; GISEL-SSE41-NEXT: movd %xmm3, %eax
+; GISEL-SSE41-NEXT: pinsrd $3, %eax, %xmm0
+; GISEL-SSE41-NEXT: retq
+;
+; GISEL-AVX-LABEL: build_vector_v4f32:
+; GISEL-AVX: # %bb.0:
+; GISEL-AVX-NEXT: vmovd %xmm0, %eax
+; GISEL-AVX-NEXT: vpinsrd $0, %eax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovd %xmm1, %eax
+; GISEL-AVX-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovd %xmm2, %eax
+; GISEL-AVX-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovd %xmm3, %eax
+; GISEL-AVX-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v4f32:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SDAG-SSE41-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; SDAG-SSE41-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v4f32:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; SDAG-AVX-NEXT: retq
+ %v0 = insertelement <4 x float> undef, float %a, i64 0
+ %v1 = insertelement <4 x float> %v0, float %b, i64 1
+ %v2 = insertelement <4 x float> %v1, float %c, i64 2
+ %v3 = insertelement <4 x float> %v2, float %d, i64 3
+ ret <4 x float> %v3
+}
+
+define <2 x double> @build_vector_v2f64(double %a, double %b) {
+; GISEL-SSE41-LABEL: build_vector_v2f64:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: movq %xmm0, %rax
+; GISEL-SSE41-NEXT: pinsrq $0, %rax, %xmm0
+; GISEL-SSE41-NEXT: movq %xmm1, %rax
+; GISEL-SSE41-NEXT: pinsrq $1, %rax, %xmm0
+; GISEL-SSE41-NEXT: retq
+;
+; GISEL-AVX-LABEL: build_vector_v2f64:
+; GISEL-AVX: # %bb.0:
+; GISEL-AVX-NEXT: vmovq %xmm0, %rax
+; GISEL-AVX-NEXT: vpinsrq $0, %rax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovq %xmm1, %rax
+; GISEL-AVX-NEXT: vpinsrq $1, %rax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v2f64:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v2f64:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SDAG-AVX-NEXT: retq
+ %v0 = insertelement <2 x double> undef, double %a, i64 0
+ %v1 = insertelement <2 x double> %v0, double %b, i64 1
+ ret <2 x double> %v1
+}
+
+;256-bit integer vectors (require AVX)
+
+define <8 x i32> @build_vector_v8i32(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h) #1 {
+; GISEL-SSE41-LABEL: build_vector_v8i32:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: vpinsrd $0, %edi, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm0, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vpinsrd $1, %esi, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vpinsrd $2, %edx, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $0, %r8d, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $1, %r9d, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v8i32:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: vmovd %edi, %xmm0
+; SDAG-SSE41-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; SDAG-SSE41-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SDAG-SSE41-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SDAG-SSE41-NEXT: vmovd %r8d, %xmm1
+; SDAG-SSE41-NEXT: vpinsrd $1, %r9d, %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; SDAG-SSE41-NEXT: retq
+ %v0 = insertelement <8 x i32> undef, i32 %a, i64 0
+ %v1 = insertelement <8 x i32> %v0, i32 %b, i64 1
+ %v2 = insertelement <8 x i32> %v1, i32 %c, i64 2
+ %v3 = insertelement <8 x i32> %v2, i32 %d, i64 3
+ %v4 = insertelement <8 x i32> %v3, i32 %e, i64 4
+ %v5 = insertelement <8 x i32> %v4, i32 %f, i64 5
+ %v6 = insertelement <8 x i32> %v5, i32 %g, i64 6
+ %v7 = insertelement <8 x i32> %v6, i32 %h, i64 7
+ ret <8 x i32> %v7
+}
+
+define <4 x i64> @build_vector_v4i64(i64 %a, i64 %b, i64 %c, i64 %d) #1 {
+; GISEL-SSE41-LABEL: build_vector_v4i64:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: vpinsrq $0, %rdi, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm0, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vpinsrq $1, %rsi, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrq $0, %rdx, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrq $1, %rcx, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v4i64:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: vmovq %rcx, %xmm0
+; SDAG-SSE41-NEXT: vmovq %rdx, %xmm1
+; SDAG-SSE41-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SDAG-SSE41-NEXT: vmovq %rsi, %xmm1
+; SDAG-SSE41-NEXT: vmovq %rdi, %xmm2
+; SDAG-SSE41-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; SDAG-SSE41-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; SDAG-SSE41-NEXT: retq
+ %v0 = insertelement <4 x i64> undef, i64 %a, i64 0
+ %v1 = insertelement <4 x i64> %v0, i64 %b, i64 1
+ %v2 = insertelement <4 x i64> %v1, i64 %c, i64 2
+ %v3 = insertelement <4 x i64> %v2, i64 %d, i64 3
+ ret <4 x i64> %v3
+}
+
+;256-bit floating-point vectors (require AVX)
+
+define <8 x float> @build_vector_v8f32(float %a, float %b, float %c, float %d, float %e, float %f, float %g, float %h) #1 {
+; GISEL-SSE41-LABEL: build_vector_v8f32:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: vmovd %xmm0, %eax
+; GISEL-SSE41-NEXT: vpinsrd $0, %eax, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm0, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vmovd %xmm1, %eax
+; GISEL-SSE41-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vmovd %xmm2, %eax
+; GISEL-SSE41-NEXT: vpinsrd $2, %eax, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vmovd %xmm3, %eax
+; GISEL-SSE41-NEXT: vpinsrd $3, %eax, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vmovd %xmm4, %eax
+; GISEL-SSE41-NEXT: vpinsrd $0, %eax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vmovd %xmm5, %eax
+; GISEL-SSE41-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vmovd %xmm6, %eax
+; GISEL-SSE41-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vmovd %xmm7, %eax
+; GISEL-SSE41-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v8f32:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[0]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; SDAG-SSE41-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v8f32:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[0]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; SDAG-AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; SDAG-AVX-NEXT: retq
+ %v0 = insertelement <8 x float> undef, float %a, i64 0
+ %v1 = insertelement <8 x float> %v0, float %b, i64 1
+ %v2 = insertelement <8 x float> %v1, float %c, i64 2
+ %v3 = insertelement <8 x float> %v2, float %d, i64 3
+ %v4 = insertelement <8 x float> %v3, float %e, i64 4
+ %v5 = insertelement <8 x float> %v4, float %f, i64 5
+ %v6 = insertelement <8 x float> %v5, float %g, i64 6
+ %v7 = insertelement <8 x float> %v6, float %h, i64 7
+ ret <8 x float> %v7
+}
+
+define <4 x double> @build_vector_v4f64(double %a, double %b, double %c, double %d) #1 {
+; GISEL-SSE41-LABEL: build_vector_v4f64:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: vmovq %xmm0, %rax
+; GISEL-SSE41-NEXT: vpinsrq $0, %rax, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm0, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vmovq %xmm1, %rax
+; GISEL-SSE41-NEXT: vpinsrq $1, %rax, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vmovq %xmm2, %rax
+; GISEL-SSE41-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: vextractf128 $1, %ymm0, %xmm1
+; GISEL-SSE41-NEXT: vmovq %xmm3, %rax
+; GISEL-SSE41-NEXT: vpinsrq $1, %rax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GISEL-SSE41-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v4f64:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SDAG-SSE41-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SDAG-SSE41-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v4f64:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SDAG-AVX-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SDAG-AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; SDAG-AVX-NEXT: retq
+ %v0 = insertelement <4 x double> undef, double %a, i64 0
+ %v1 = insertelement <4 x double> %v0, double %b, i64 1
+ %v2 = insertelement <4 x double> %v1, double %c, i64 2
+ %v3 = insertelement <4 x double> %v2, double %d, i64 3
+ ret <4 x double> %v3
+}
+
+;512-bit integer vectors (require AVX512)
+
+define <16 x i32> @build_vector_v16i32(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h,
+; GISEL-SSE41-LABEL: build_vector_v16i32:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: vpinsrd $0, %edi, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vpinsrd $1, %esi, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vpinsrd $2, %edx, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $0, %r8d, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $1, %r9d, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: retq
+;
+; GISEL-AVX-LABEL: build_vector_v16i32:
+; GISEL-AVX: # %bb.0:
+; GISEL-AVX-NEXT: vpinsrd $0, %edi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vpinsrd $1, %esi, %xmm0, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vpinsrd $2, %edx, %xmm0, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $0, %r8d, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $1, %r9d, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v16i32:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: vmovd %edi, %xmm0
+; SDAG-SSE41-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; SDAG-SSE41-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SDAG-SSE41-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SDAG-SSE41-NEXT: vmovd %r8d, %xmm1
+; SDAG-SSE41-NEXT: vpinsrd $1, %r9d, %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; SDAG-SSE41-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SDAG-SSE41-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-SSE41-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SDAG-SSE41-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; SDAG-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; SDAG-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; SDAG-SSE41-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; SDAG-SSE41-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v16i32:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vmovd %edi, %xmm0
+; SDAG-AVX-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; SDAG-AVX-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SDAG-AVX-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SDAG-AVX-NEXT: vmovd %r8d, %xmm1
+; SDAG-AVX-NEXT: vpinsrd $1, %r9d, %xmm1, %xmm1
+; SDAG-AVX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-AVX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-AVX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; SDAG-AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SDAG-AVX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-AVX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-AVX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; SDAG-AVX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SDAG-AVX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; SDAG-AVX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; SDAG-AVX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; SDAG-AVX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; SDAG-AVX-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; SDAG-AVX-NEXT: retq
+ i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p) #2 {
+ %v0 = insertelement <16 x i32> undef, i32 %a, i64 0
+ %v1 = insertelement <16 x i32> %v0, i32 %b, i64 1
+ %v2 = insertelement <16 x i32> %v1, i32 %c, i64 2
+ %v3 = insertelement <16 x i32> %v2, i32 %d, i64 3
+ %v4 = insertelement <16 x i32> %v3, i32 %e, i64 4
+ %v5 = insertelement <16 x i32> %v4, i32 %f, i64 5
+ %v6 = insertelement <16 x i32> %v5, i32 %g, i64 6
+ %v7 = insertelement <16 x i32> %v6, i32 %h, i64 7
+ %v8 = insertelement <16 x i32> %v7, i32 %i, i64 8
+ %v9 = insertelement <16 x i32> %v8, i32 %j, i64 9
+ %v10 = insertelement <16 x i32> %v9, i32 %k, i64 10
+ %v11 = insertelement <16 x i32> %v10, i32 %l, i64 11
+ %v12 = insertelement <16 x i32> %v11, i32 %m, i64 12
+ %v13 = insertelement <16 x i32> %v12, i32 %n, i64 13
+ %v14 = insertelement <16 x i32> %v13, i32 %o, i64 14
+ %v15 = insertelement <16 x i32> %v14, i32 %p, i64 15
+ ret <16 x i32> %v15
+}
+
+define <8 x i64> @build_vector_v8i64(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64 %f, i64 %g, i64 %h) #2 {
+; GISEL-SSE41-LABEL: build_vector_v8i64:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: vpinsrq $0, %rdi, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vpinsrq $1, %rsi, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrq $0, %rdx, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrq $1, %rcx, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrq $0, %r8, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrq $1, %r9, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrq $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrq $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: retq
+;
+; GISEL-AVX-LABEL: build_vector_v8i64:
+; GISEL-AVX: # %bb.0:
+; GISEL-AVX-NEXT: vpinsrq $0, %rdi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vpinsrq $1, %rsi, %xmm0, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrq $0, %rdx, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrq $1, %rcx, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrq $0, %r8, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrq $1, %r9, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrq $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrq $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v8i64:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: vmovq %rcx, %xmm0
+; SDAG-SSE41-NEXT: vmovq %rdx, %xmm1
+; SDAG-SSE41-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SDAG-SSE41-NEXT: vmovq %rsi, %xmm1
+; SDAG-SSE41-NEXT: vmovq %rdi, %xmm2
+; SDAG-SSE41-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; SDAG-SSE41-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; SDAG-SSE41-NEXT: vmovq %r9, %xmm1
+; SDAG-SSE41-NEXT: vmovq %r8, %xmm2
+; SDAG-SSE41-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; SDAG-SSE41-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
+; SDAG-SSE41-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v8i64:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vmovq %rcx, %xmm0
+; SDAG-AVX-NEXT: vmovq %rdx, %xmm1
+; SDAG-AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SDAG-AVX-NEXT: vmovq %rsi, %xmm1
+; SDAG-AVX-NEXT: vmovq %rdi, %xmm2
+; SDAG-AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; SDAG-AVX-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; SDAG-AVX-NEXT: vmovq %r9, %xmm1
+; SDAG-AVX-NEXT: vmovq %r8, %xmm2
+; SDAG-AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; SDAG-AVX-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
+; SDAG-AVX-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; SDAG-AVX-NEXT: retq
+ %v0 = insertelement <8 x i64> undef, i64 %a, i64 0
+ %v1 = insertelement <8 x i64> %v0, i64 %b, i64 1
+ %v2 = insertelement <8 x i64> %v1, i64 %c, i64 2
+ %v3 = insertelement <8 x i64> %v2, i64 %d, i64 3
+ %v4 = insertelement <8 x i64> %v3, i64 %e, i64 4
+ %v5 = insertelement <8 x i64> %v4, i64 %f, i64 5
+ %v6 = insertelement <8 x i64> %v5, i64 %g, i64 6
+ %v7 = insertelement <8 x i64> %v6, i64 %h, i64 7
+ ret <8 x i64> %v7
+}
+
+;512-bit floating-point vectors (require AVX512)
+
+define <16 x float> @build_vector_v16f32(float %a, float %b, float %c, float %d, float %e, float %f, float %g, float %h,
+; GISEL-SSE41-LABEL: build_vector_v16f32:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: vmovd %xmm0, %eax
+; GISEL-SSE41-NEXT: vpinsrd $0, %eax, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vmovd %xmm1, %eax
+; GISEL-SSE41-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vmovd %xmm2, %eax
+; GISEL-SSE41-NEXT: vpinsrd $2, %eax, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vmovd %xmm3, %eax
+; GISEL-SSE41-NEXT: vpinsrd $3, %eax, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovd %xmm4, %eax
+; GISEL-SSE41-NEXT: vpinsrd $0, %eax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovd %xmm5, %eax
+; GISEL-SSE41-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovd %xmm6, %eax
+; GISEL-SSE41-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovd %xmm7, %eax
+; GISEL-SSE41-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: retq
+;
+; GISEL-AVX-LABEL: build_vector_v16f32:
+; GISEL-AVX: # %bb.0:
+; GISEL-AVX-NEXT: vmovd %xmm0, %eax
+; GISEL-AVX-NEXT: vpinsrd $0, %eax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vmovd %xmm1, %eax
+; GISEL-AVX-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vmovd %xmm2, %eax
+; GISEL-AVX-NEXT: vpinsrd $2, %eax, %xmm0, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vmovd %xmm3, %eax
+; GISEL-AVX-NEXT: vpinsrd $3, %eax, %xmm0, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovd %xmm4, %eax
+; GISEL-AVX-NEXT: vpinsrd $0, %eax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovd %xmm5, %eax
+; GISEL-AVX-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovd %xmm6, %eax
+; GISEL-AVX-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovd %xmm7, %eax
+; GISEL-AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $0, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v16f32:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[0]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; SDAG-SSE41-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; SDAG-SSE41-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
+; SDAG-SSE41-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; SDAG-SSE41-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
+; SDAG-SSE41-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; SDAG-SSE41-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v16f32:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[0]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; SDAG-AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; SDAG-AVX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
+; SDAG-AVX-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; SDAG-AVX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
+; SDAG-AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; SDAG-AVX-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; SDAG-AVX-NEXT: retq
+ float %i, float %j, float %k, float %l, float %m, float %n, float %o, float %p) #2 {
+ %v0 = insertelement <16 x float> undef, float %a, i64 0
+ %v1 = insertelement <16 x float> %v0, float %b, i64 1
+ %v2 = insertelement <16 x float> %v1, float %c, i64 2
+ %v3 = insertelement <16 x float> %v2, float %d, i64 3
+ %v4 = insertelement <16 x float> %v3, float %e, i64 4
+ %v5 = insertelement <16 x float> %v4, float %f, i64 5
+ %v6 = insertelement <16 x float> %v5, float %g, i64 6
+ %v7 = insertelement <16 x float> %v6, float %h, i64 7
+ %v8 = insertelement <16 x float> %v7, float %i, i64 8
+ %v9 = insertelement <16 x float> %v8, float %j, i64 9
+ %v10 = insertelement <16 x float> %v9, float %k, i64 10
+ %v11 = insertelement <16 x float> %v10, float %l, i64 11
+ %v12 = insertelement <16 x float> %v11, float %m, i64 12
+ %v13 = insertelement <16 x float> %v12, float %n, i64 13
+ %v14 = insertelement <16 x float> %v13, float %o, i64 14
+ %v15 = insertelement <16 x float> %v14, float %p, i64 15
+ ret <16 x float> %v15
+}
+
+define <8 x double> @build_vector_v8f64(double %a, double %b, double %c, double %d, double %e, double %f, double %g, double %h) #2 {
+; GISEL-SSE41-LABEL: build_vector_v8f64:
+; GISEL-SSE41: # %bb.0:
+; GISEL-SSE41-NEXT: vmovq %xmm0, %rax
+; GISEL-SSE41-NEXT: vpinsrq $0, %rax, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vmovq %xmm1, %rax
+; GISEL-SSE41-NEXT: vpinsrq $1, %rax, %xmm0, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovq %xmm2, %rax
+; GISEL-SSE41-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovq %xmm3, %rax
+; GISEL-SSE41-NEXT: vpinsrq $1, %rax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovq %xmm4, %rax
+; GISEL-SSE41-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovq %xmm5, %rax
+; GISEL-SSE41-NEXT: vpinsrq $1, %rax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovq %xmm6, %rax
+; GISEL-SSE41-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-SSE41-NEXT: vmovq %xmm7, %rax
+; GISEL-SSE41-NEXT: vpinsrq $1, %rax, %xmm1, %xmm1
+; GISEL-SSE41-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-SSE41-NEXT: retq
+;
+; GISEL-AVX-LABEL: build_vector_v8f64:
+; GISEL-AVX: # %bb.0:
+; GISEL-AVX-NEXT: vmovq %xmm0, %rax
+; GISEL-AVX-NEXT: vpinsrq $0, %rax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vmovq %xmm1, %rax
+; GISEL-AVX-NEXT: vpinsrq $1, %rax, %xmm0, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovq %xmm2, %rax
+; GISEL-AVX-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $1, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovq %xmm3, %rax
+; GISEL-AVX-NEXT: vpinsrq $1, %rax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovq %xmm4, %rax
+; GISEL-AVX-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovq %xmm5, %rax
+; GISEL-AVX-NEXT: vpinsrq $1, %rax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovq %xmm6, %rax
+; GISEL-AVX-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; GISEL-AVX-NEXT: vmovq %xmm7, %rax
+; GISEL-AVX-NEXT: vpinsrq $1, %rax, %xmm1, %xmm1
+; GISEL-AVX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0
+; GISEL-AVX-NEXT: retq
+;
+; SDAG-SSE41-LABEL: build_vector_v8f64:
+; SDAG-SSE41: # %bb.0:
+; SDAG-SSE41-NEXT: vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
+; SDAG-SSE41-NEXT: vmovlhps {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SDAG-SSE41-NEXT: vinsertf128 $1, %xmm6, %ymm4, %ymm4
+; SDAG-SSE41-NEXT: vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SDAG-SSE41-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SDAG-SSE41-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; SDAG-SSE41-NEXT: vinsertf64x4 $1, %ymm4, %zmm0, %zmm0
+; SDAG-SSE41-NEXT: retq
+;
+; SDAG-AVX-LABEL: build_vector_v8f64:
+; SDAG-AVX: # %bb.0:
+; SDAG-AVX-NEXT: vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
+; SDAG-AVX-NEXT: vmovlhps {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SDAG-AVX-NEXT: vinsertf128 $1, %xmm6, %ymm4, %ymm4
+; SDAG-AVX-NEXT: vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SDAG-AVX-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SDAG-AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; SDAG-AVX-NEXT: vinsertf64x4 $1, %ymm4, %zmm0, %zmm0
+; SDAG-AVX-NEXT: retq
+ %v0 = insertelement <8 x double> undef, double %a, i64 0
+ %v1 = insertelement <8 x double> %v0, double %b, i64 1
+ %v2 = insertelement <8 x double> %v1, double %c, i64 2
+ %v3 = insertelement <8 x double> %v2, double %d, i64 3
+ %v4 = insertelement <8 x double> %v3, double %e, i64 4
+ %v5 = insertelement <8 x double> %v4, double %f, i64 5
+ %v6 = insertelement <8 x double> %v5, double %g, i64 6
+ %v7 = insertelement <8 x double> %v6, double %h, i64 7
+ ret <8 x double> %v7
+}
+
+attributes #1 = { "target-features"="+avx" }
+attributes #2 = { "target-features"="+avx512f,+avx512bw,+avx512dq" }
>From e9ff617232160de5a1ddd29fc69ccab1f650f271 Mon Sep 17 00:00:00 2001
From: mattarde <mattarde at intel.com>
Date: Tue, 30 Jun 2026 01:20:31 -0700
Subject: [PATCH 2/2] add break dep for vpnisert
---
.../X86/GISel/X86InstructionSelector.cpp | 27 ++++++++++++-
.../X86/GISel/X86PostLegalizerCombiner.cpp | 33 +++++++++++++++
.../Target/X86/GISel/X86RegisterBankInfo.cpp | 1 +
llvm/lib/Target/X86/X86Combine.td | 12 +++++-
llvm/lib/Target/X86/X86InstrGISel.td | 6 +++
.../CodeGen/X86/GlobalISel/build-vector.ll | 40 +++++++++----------
6 files changed, 96 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
index ae8ad6823acb1..8470e07b128bc 100644
--- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
+++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
@@ -123,6 +123,7 @@ class X86InstructionSelector : public InstructionSelector {
MachineFunction &MF) const;
bool selectInsertVectorElt(MachineInstr &I, MachineRegisterInfo &MRI,
MachineFunction &MF);
+ bool selectScalarToVector(MachineInstr &I, MachineRegisterInfo &MRI);
unsigned getMachineOpcodeForInsert(unsigned EltSizeInBits) const;
unsigned getScalarToVecOpcode(unsigned EltSizeInBits) const;
@@ -498,6 +499,8 @@ bool X86InstructionSelector::select(MachineInstr &I) {
return selectSelect(I, MRI, MF);
case TargetOpcode::G_INSERT_VECTOR_ELT:
return selectInsertVectorElt(I, MRI, MF);
+ case X86::G_SCALAR_TO_VECTOR:
+ return selectScalarToVector(I, MRI);
}
return false;
@@ -1995,7 +1998,7 @@ unsigned X86InstructionSelector::getMachineOpcodeForInsert(
{X86::PINSRQrri, X86::VPINSRQrri, X86::VPINSRQZrri}, // 64-bit
};
assert(EltSizeInBits % 8 == 0 && EltSizeInBits <= 64);
- unsigned Row = EltSizeInBits / 8 - 1;
+ unsigned Row = Log2_32(EltSizeInBits / 8);
if (EltSizeInBits == 8) {
if (HasBWI)
return PinsrOpcodes[Row][2];
@@ -2063,7 +2066,6 @@ bool X86InstructionSelector::selectInsertVectorElt(MachineInstr &I,
if (Idx >= NumElts)
return false;
- // TODO: use movd/vmovd/movq at idx 0 to avoid false dependency.
unsigned Opc = getMachineOpcodeForInsert(EltSize);
if (!Opc)
@@ -2079,6 +2081,27 @@ bool X86InstructionSelector::selectInsertVectorElt(MachineInstr &I,
return true;
}
+bool X86InstructionSelector::selectScalarToVector(MachineInstr &I,
+ MachineRegisterInfo &MRI) {
+ assert(I.getOpcode() == X86::G_SCALAR_TO_VECTOR);
+
+ Register DstReg = I.getOperand(0).getReg();
+ Register SrcReg = I.getOperand(1).getReg();
+ LLT DstTy = MRI.getType(DstReg);
+ unsigned EltSize = DstTy.getScalarSizeInBits();
+
+ unsigned Opc = getScalarToVecOpcode(EltSize);
+ if (!Opc)
+ return false;
+
+ auto &MIB = *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opc), DstReg)
+ .addReg(SrcReg);
+
+ constrainSelectedInstRegOperands(MIB, TII, TRI, RBI);
+ I.eraseFromParent();
+ return true;
+}
+
InstructionSelector::ComplexRendererFns
X86InstructionSelector::selectAddr(MachineOperand &Root) const {
MachineInstr *MI = Root.getParent();
diff --git a/llvm/lib/Target/X86/GISel/X86PostLegalizerCombiner.cpp b/llvm/lib/Target/X86/GISel/X86PostLegalizerCombiner.cpp
index 411cc190dbaff..c523067f82241 100644
--- a/llvm/lib/Target/X86/GISel/X86PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/X86/GISel/X86PostLegalizerCombiner.cpp
@@ -51,6 +51,39 @@ using namespace MIPatternMatch;
namespace {
+// Match G_INSERT_VECTOR_ELT(G_IMPLICIT_DEF, elt, 0) where element size is
+// 32 or 64 bits. This pattern can be replaced with G_SCALAR_TO_VECTOR which
+// selects to movd/movq, avoiding the false dependency that pinsrd/pinsrq
+// would introduce on the (undefined) destination register.
+static bool matchInsertVecEltToScalarToVec(MachineInstr &MI,
+ MachineRegisterInfo &MRI) {
+ assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
+
+ Register VecReg = MI.getOperand(1).getReg();
+ Register IdxReg = MI.getOperand(3).getReg();
+
+ if (!getOpcodeDef<GImplicitDef>(VecReg, MRI))
+ return false;
+
+ auto IdxVal = getIConstantVRegValWithLookThrough(IdxReg, MRI);
+ if (!IdxVal || IdxVal->Value.getZExtValue() != 0)
+ return false;
+
+ LLT VecTy = MRI.getType(MI.getOperand(0).getReg());
+ unsigned EltSize = VecTy.getScalarSizeInBits();
+ return EltSize == 32 || EltSize == 64;
+}
+
+static void applyInsertVecEltToScalarToVec(MachineInstr &MI,
+ MachineRegisterInfo &MRI,
+ MachineIRBuilder &B) {
+ B.setInstrAndDebugLoc(MI);
+ Register Dst = MI.getOperand(0).getReg();
+ Register Elt = MI.getOperand(2).getReg();
+ B.buildInstr(X86::G_SCALAR_TO_VECTOR, {Dst}, {Elt});
+ MI.eraseFromParent();
+}
+
CombinerInfo createCombinerInfo(bool OptEnabled, const Function &F) {
CombinerInfo CInfo(/*AllowIllegalOps=*/true,
/*ShouldLegalizeIllegal=*/false,
diff --git a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
index a0c4d3a2633aa..d2ee186e89091 100644
--- a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
@@ -301,6 +301,7 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
// VECRReg)
getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ true, OpRegBankIdx);
break;
+ case X86::G_SCALAR_TO_VECTOR:
case X86::G_FIST:
case X86::G_FILD: {
auto &Op0 = MI.getOperand(0);
diff --git a/llvm/lib/Target/X86/X86Combine.td b/llvm/lib/Target/X86/X86Combine.td
index b6d010c41e25f..681ae635d9a15 100644
--- a/llvm/lib/Target/X86/X86Combine.td
+++ b/llvm/lib/Target/X86/X86Combine.td
@@ -18,6 +18,16 @@ def all_x86combines : GICombineGroup<[identity_combines, reassocs,
def X86PreLegalizerCombiner : GICombiner<"X86PreLegalizerCombinerImpl", [all_x86combines]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
-def X86PostLegalizerCombiner : GICombiner<"X86PostLegalizerCombinerImpl", [redundant_or, constant_fold_fp_ops, identity_combines, copy_prop]> {
+
+def insert_vec_elt_to_scalar_to_vec : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
+ [{ return matchInsertVecEltToScalarToVec(*${root}, MRI); }]),
+ (apply [{ applyInsertVecEltToScalarToVec(*${root}, MRI, B); }])>;
+
+def X86PostLegalizerCombiner : GICombiner<
+ "X86PostLegalizerCombinerImpl",
+ [redundant_or, constant_fold_fp_ops, identity_combines, copy_prop,
+ insert_vec_elt_to_scalar_to_vec]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/X86/X86InstrGISel.td b/llvm/lib/Target/X86/X86InstrGISel.td
index 521923175c444..5a79223c4fea8 100644
--- a/llvm/lib/Target/X86/X86InstrGISel.td
+++ b/llvm/lib/Target/X86/X86InstrGISel.td
@@ -49,6 +49,12 @@ def G_WRAPPER_RIP : X86GenericInstruction {
let hasSideEffects = false;
}
+def G_SCALAR_TO_VECTOR : X86GenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$src);
+ let hasSideEffects = false;
+}
+
def : GINodeEquiv<G_FILD, X86fild>;
def : GINodeEquiv<G_FIST, X86fp_to_mem>;
def : GINodeEquiv<G_FNSTCW16, X86fp_cwd_get16>;
diff --git a/llvm/test/CodeGen/X86/GlobalISel/build-vector.ll b/llvm/test/CodeGen/X86/GlobalISel/build-vector.ll
index fc103ff64995f..ff632ff5888bc 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/build-vector.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/build-vector.ll
@@ -12,7 +12,7 @@
define <4 x i32> @build_vector_v4i32(i32 %a, i32 %b, i32 %c, i32 %d) {
; GISEL-SSE41-LABEL: build_vector_v4i32:
; GISEL-SSE41: # %bb.0:
-; GISEL-SSE41-NEXT: pinsrd $0, %edi, %xmm0
+; GISEL-SSE41-NEXT: movd %edi, %xmm0
; GISEL-SSE41-NEXT: pinsrd $1, %esi, %xmm0
; GISEL-SSE41-NEXT: pinsrd $2, %edx, %xmm0
; GISEL-SSE41-NEXT: pinsrd $3, %ecx, %xmm0
@@ -20,7 +20,7 @@ define <4 x i32> @build_vector_v4i32(i32 %a, i32 %b, i32 %c, i32 %d) {
;
; GISEL-AVX-LABEL: build_vector_v4i32:
; GISEL-AVX: # %bb.0:
-; GISEL-AVX-NEXT: vpinsrd $0, %edi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovd %edi, %xmm0
; GISEL-AVX-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
; GISEL-AVX-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; GISEL-AVX-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
@@ -51,13 +51,13 @@ define <4 x i32> @build_vector_v4i32(i32 %a, i32 %b, i32 %c, i32 %d) {
define <2 x i64> @build_vector_v2i64(i64 %a, i64 %b) {
; GISEL-SSE41-LABEL: build_vector_v2i64:
; GISEL-SSE41: # %bb.0:
-; GISEL-SSE41-NEXT: pinsrq $0, %rdi, %xmm0
+; GISEL-SSE41-NEXT: movq %rdi, %xmm0
; GISEL-SSE41-NEXT: pinsrq $1, %rsi, %xmm0
; GISEL-SSE41-NEXT: retq
;
; GISEL-AVX-LABEL: build_vector_v2i64:
; GISEL-AVX: # %bb.0:
-; GISEL-AVX-NEXT: vpinsrq $0, %rdi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovq %rdi, %xmm0
; GISEL-AVX-NEXT: vpinsrq $1, %rsi, %xmm0, %xmm0
; GISEL-AVX-NEXT: retq
;
@@ -85,7 +85,7 @@ define <4 x float> @build_vector_v4f32(float %a, float %b, float %c, float %d) {
; GISEL-SSE41-LABEL: build_vector_v4f32:
; GISEL-SSE41: # %bb.0:
; GISEL-SSE41-NEXT: movd %xmm0, %eax
-; GISEL-SSE41-NEXT: pinsrd $0, %eax, %xmm0
+; GISEL-SSE41-NEXT: movd %eax, %xmm0
; GISEL-SSE41-NEXT: movd %xmm1, %eax
; GISEL-SSE41-NEXT: pinsrd $1, %eax, %xmm0
; GISEL-SSE41-NEXT: movd %xmm2, %eax
@@ -97,7 +97,7 @@ define <4 x float> @build_vector_v4f32(float %a, float %b, float %c, float %d) {
; GISEL-AVX-LABEL: build_vector_v4f32:
; GISEL-AVX: # %bb.0:
; GISEL-AVX-NEXT: vmovd %xmm0, %eax
-; GISEL-AVX-NEXT: vpinsrd $0, %eax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovd %eax, %xmm0
; GISEL-AVX-NEXT: vmovd %xmm1, %eax
; GISEL-AVX-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
; GISEL-AVX-NEXT: vmovd %xmm2, %eax
@@ -130,7 +130,7 @@ define <2 x double> @build_vector_v2f64(double %a, double %b) {
; GISEL-SSE41-LABEL: build_vector_v2f64:
; GISEL-SSE41: # %bb.0:
; GISEL-SSE41-NEXT: movq %xmm0, %rax
-; GISEL-SSE41-NEXT: pinsrq $0, %rax, %xmm0
+; GISEL-SSE41-NEXT: movq %rax, %xmm0
; GISEL-SSE41-NEXT: movq %xmm1, %rax
; GISEL-SSE41-NEXT: pinsrq $1, %rax, %xmm0
; GISEL-SSE41-NEXT: retq
@@ -138,7 +138,7 @@ define <2 x double> @build_vector_v2f64(double %a, double %b) {
; GISEL-AVX-LABEL: build_vector_v2f64:
; GISEL-AVX: # %bb.0:
; GISEL-AVX-NEXT: vmovq %xmm0, %rax
-; GISEL-AVX-NEXT: vpinsrq $0, %rax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovq %rax, %xmm0
; GISEL-AVX-NEXT: vmovq %xmm1, %rax
; GISEL-AVX-NEXT: vpinsrq $1, %rax, %xmm0, %xmm0
; GISEL-AVX-NEXT: retq
@@ -162,7 +162,7 @@ define <2 x double> @build_vector_v2f64(double %a, double %b) {
define <8 x i32> @build_vector_v8i32(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h) #1 {
; GISEL-SSE41-LABEL: build_vector_v8i32:
; GISEL-SSE41: # %bb.0:
-; GISEL-SSE41-NEXT: vpinsrd $0, %edi, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vmovd %edi, %xmm0
; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm0, %ymm0, %ymm0
; GISEL-SSE41-NEXT: vpinsrd $1, %esi, %xmm0, %xmm1
; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
@@ -210,7 +210,7 @@ define <8 x i32> @build_vector_v8i32(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32
define <4 x i64> @build_vector_v4i64(i64 %a, i64 %b, i64 %c, i64 %d) #1 {
; GISEL-SSE41-LABEL: build_vector_v4i64:
; GISEL-SSE41: # %bb.0:
-; GISEL-SSE41-NEXT: vpinsrq $0, %rdi, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vmovq %rdi, %xmm0
; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm0, %ymm0, %ymm0
; GISEL-SSE41-NEXT: vpinsrq $1, %rsi, %xmm0, %xmm1
; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm1, %ymm0, %ymm0
@@ -245,7 +245,7 @@ define <8 x float> @build_vector_v8f32(float %a, float %b, float %c, float %d, f
; GISEL-SSE41-LABEL: build_vector_v8f32:
; GISEL-SSE41: # %bb.0:
; GISEL-SSE41-NEXT: vmovd %xmm0, %eax
-; GISEL-SSE41-NEXT: vpinsrd $0, %eax, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vmovd %eax, %xmm0
; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm0, %ymm0, %ymm0
; GISEL-SSE41-NEXT: vmovd %xmm1, %eax
; GISEL-SSE41-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
@@ -310,7 +310,7 @@ define <4 x double> @build_vector_v4f64(double %a, double %b, double %c, double
; GISEL-SSE41-LABEL: build_vector_v4f64:
; GISEL-SSE41: # %bb.0:
; GISEL-SSE41-NEXT: vmovq %xmm0, %rax
-; GISEL-SSE41-NEXT: vpinsrq $0, %rax, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vmovq %rax, %xmm0
; GISEL-SSE41-NEXT: vinsertf128 $0, %xmm0, %ymm0, %ymm0
; GISEL-SSE41-NEXT: vmovq %xmm1, %rax
; GISEL-SSE41-NEXT: vpinsrq $1, %rax, %xmm0, %xmm1
@@ -350,7 +350,7 @@ define <4 x double> @build_vector_v4f64(double %a, double %b, double %c, double
define <16 x i32> @build_vector_v16i32(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h,
; GISEL-SSE41-LABEL: build_vector_v16i32:
; GISEL-SSE41: # %bb.0:
-; GISEL-SSE41-NEXT: vpinsrd $0, %edi, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vmovd %edi, %xmm0
; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
; GISEL-SSE41-NEXT: vpinsrd $1, %esi, %xmm0, %xmm1
; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
@@ -398,7 +398,7 @@ define <16 x i32> @build_vector_v16i32(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i
;
; GISEL-AVX-LABEL: build_vector_v16i32:
; GISEL-AVX: # %bb.0:
-; GISEL-AVX-NEXT: vpinsrd $0, %edi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovd %edi, %xmm0
; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
; GISEL-AVX-NEXT: vpinsrd $1, %esi, %xmm0, %xmm1
; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
@@ -512,7 +512,7 @@ define <16 x i32> @build_vector_v16i32(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i
define <8 x i64> @build_vector_v8i64(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64 %f, i64 %g, i64 %h) #2 {
; GISEL-SSE41-LABEL: build_vector_v8i64:
; GISEL-SSE41: # %bb.0:
-; GISEL-SSE41-NEXT: vpinsrq $0, %rdi, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vmovq %rdi, %xmm0
; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
; GISEL-SSE41-NEXT: vpinsrq $1, %rsi, %xmm0, %xmm1
; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
@@ -538,7 +538,7 @@ define <8 x i64> @build_vector_v8i64(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64
;
; GISEL-AVX-LABEL: build_vector_v8i64:
; GISEL-AVX: # %bb.0:
-; GISEL-AVX-NEXT: vpinsrq $0, %rdi, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovq %rdi, %xmm0
; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
; GISEL-AVX-NEXT: vpinsrq $1, %rsi, %xmm0, %xmm1
; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
@@ -610,7 +610,7 @@ define <16 x float> @build_vector_v16f32(float %a, float %b, float %c, float %d,
; GISEL-SSE41-LABEL: build_vector_v16f32:
; GISEL-SSE41: # %bb.0:
; GISEL-SSE41-NEXT: vmovd %xmm0, %eax
-; GISEL-SSE41-NEXT: vpinsrd $0, %eax, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vmovd %eax, %xmm0
; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
; GISEL-SSE41-NEXT: vmovd %xmm1, %eax
; GISEL-SSE41-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
@@ -666,7 +666,7 @@ define <16 x float> @build_vector_v16f32(float %a, float %b, float %c, float %d,
; GISEL-AVX-LABEL: build_vector_v16f32:
; GISEL-AVX: # %bb.0:
; GISEL-AVX-NEXT: vmovd %xmm0, %eax
-; GISEL-AVX-NEXT: vpinsrd $0, %eax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovd %eax, %xmm0
; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
; GISEL-AVX-NEXT: vmovd %xmm1, %eax
; GISEL-AVX-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
@@ -784,7 +784,7 @@ define <8 x double> @build_vector_v8f64(double %a, double %b, double %c, double
; GISEL-SSE41-LABEL: build_vector_v8f64:
; GISEL-SSE41: # %bb.0:
; GISEL-SSE41-NEXT: vmovq %xmm0, %rax
-; GISEL-SSE41-NEXT: vpinsrq $0, %rax, %xmm0, %xmm0
+; GISEL-SSE41-NEXT: vmovq %rax, %xmm0
; GISEL-SSE41-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
; GISEL-SSE41-NEXT: vmovq %xmm1, %rax
; GISEL-SSE41-NEXT: vpinsrq $1, %rax, %xmm0, %xmm1
@@ -818,7 +818,7 @@ define <8 x double> @build_vector_v8f64(double %a, double %b, double %c, double
; GISEL-AVX-LABEL: build_vector_v8f64:
; GISEL-AVX: # %bb.0:
; GISEL-AVX-NEXT: vmovq %xmm0, %rax
-; GISEL-AVX-NEXT: vpinsrq $0, %rax, %xmm0, %xmm0
+; GISEL-AVX-NEXT: vmovq %rax, %xmm0
; GISEL-AVX-NEXT: vinserti32x4 $0, %xmm0, %zmm0, %zmm0
; GISEL-AVX-NEXT: vmovq %xmm1, %rax
; GISEL-AVX-NEXT: vpinsrq $1, %rax, %xmm0, %xmm1
More information about the llvm-commits
mailing list