[llvm] 5fa41e2 - [X86] Use valign instead of vperm for float domain shuffles (#201624)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 07:09:34 PDT 2026
Author: Ganesh
Date: 2026-06-26T19:39:29+05:30
New Revision: 5fa41e26aaa01fb79c4c5d7ff71f027757e444ae
URL: https://github.com/llvm/llvm-project/commit/5fa41e26aaa01fb79c4c5d7ff71f027757e444ae
DIFF: https://github.com/llvm/llvm-project/commit/5fa41e26aaa01fb79c4c5d7ff71f027757e444ae.diff
LOG: [X86] Use valign instead of vperm for float domain shuffles (#201624)
The X86 backend then lowers the shuffle through lowerV16F32Shuffle /
lowerV8F64Shuffle, which fall through to lowerShuffleWithPERMV (VPERMPS
/ VPERMPD). lowerShuffleAsVALIGN is asserted on i32 / i64 element types
only and is never called from the float-domain paths, even when the mask
is a clean concatenate-and-shift that VALIGN expresses exactly.
On znver5, VALIGN and VPERMPS / VPERMPD have identical latency (5 cycles
for zmm), throughput (2), and macro-op count (1). The real cost of
VPERMPS / VPERMPD is the extra zmm register required to hold the
permutation index vector.
Intrinsic path for _mm512_alignr_epi32 also gets a vperm. Its a win in
generic path as well as vpermps zmm1, zmm0, zmm3 requires a dedicated
zmm register to hold the permutation index vector. valignd zmm1, zmm3,
zmm3, 1 encodes the rotation count as an immediate (imm8 = 1), using no
extra registers.
Co-authored-by: Shivanshu
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index cf4e0c081a1b8..b95ac78f50049 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -12673,18 +12673,29 @@ static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
const APInt &Zeroable,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
- assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
- "Only 32-bit and 64-bit elements are supported!");
+ unsigned EltBits = VT.getScalarSizeInBits();
+ if (EltBits != 32 && EltBits != 64)
+ return SDValue();
+
+ MVT AlignVT = VT.changeVectorElementTypeToInteger();
// 128/256-bit vectors are only supported with VLX.
- assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector()))
- && "VLX required for 128/256-bit vectors");
+ assert((Subtarget.hasVLX() ||
+ (!AlignVT.is128BitVector() && !AlignVT.is256BitVector())) &&
+ "VLX required for 128/256-bit vectors");
+
+ auto emitVALIGN = [&](SDValue Lo, SDValue Hi, unsigned Imm) -> SDValue {
+ SDValue AlignLo = VT.isFloatingPoint() ? DAG.getBitcast(AlignVT, Lo) : Lo;
+ SDValue AlignHi = VT.isFloatingPoint() ? DAG.getBitcast(AlignVT, Hi) : Hi;
+ SDValue Res = DAG.getNode(X86ISD::VALIGN, DL, AlignVT, AlignLo, AlignHi,
+ DAG.getTargetConstant(Imm, DL, MVT::i8));
+ return VT.isFloatingPoint() ? DAG.getBitcast(VT, Res) : Res;
+ };
SDValue Lo = V1, Hi = V2;
int Rotation = matchShuffleAsElementRotate(Lo, Hi, Mask);
if (0 < Rotation)
- return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
- DAG.getTargetConstant(Rotation, DL, MVT::i8));
+ return emitVALIGN(Lo, Hi, Rotation);
// See if we can use VALIGN as a cross-lane version of VSHLDQ/VSRLDQ.
// TODO: Pull this out as a matchShuffleAsElementShift helper?
@@ -12701,18 +12712,16 @@ static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
SDValue Src = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
int Low = Mask[ZeroLo] < (int)NumElts ? 0 : NumElts;
if (isSequentialOrUndefInRange(Mask, ZeroLo, NumElts - ZeroLo, Low))
- return DAG.getNode(X86ISD::VALIGN, DL, VT, Src,
- getZeroVector(VT, Subtarget, DAG, DL),
- DAG.getTargetConstant(NumElts - ZeroLo, DL, MVT::i8));
+ return emitVALIGN(Src, getZeroVector(AlignVT, Subtarget, DAG, DL),
+ NumElts - ZeroLo);
}
if (ZeroHi) {
SDValue Src = Mask[0] < (int)NumElts ? V1 : V2;
int Low = Mask[0] < (int)NumElts ? 0 : NumElts;
if (isSequentialOrUndefInRange(Mask, 0, NumElts - ZeroHi, Low + ZeroHi))
- return DAG.getNode(X86ISD::VALIGN, DL, VT,
- getZeroVector(VT, Subtarget, DAG, DL), Src,
- DAG.getTargetConstant(ZeroHi, DL, MVT::i8));
+ return emitVALIGN(getZeroVector(AlignVT, Subtarget, DAG, DL), Src,
+ ZeroHi);
}
return SDValue();
@@ -18111,6 +18120,12 @@ static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
Zeroable, Subtarget, DAG))
return Blend;
+ // Try to use VALIGN via integer domain bitcast. Avoids VPERMPD which
+ // requires an extra register for the index vector; VALIGNQ uses an immediate.
+ if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8f64, V1, V2, Mask,
+ Zeroable, Subtarget, DAG))
+ return Rotate;
+
return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG);
}
@@ -18178,6 +18193,12 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
Zeroable, Subtarget, DAG))
return V;
+ // Try to use VALIGN via integer domain bitcast. Avoids VPERMPS which
+ // requires an extra register for the index vector; VALIGND uses an immediate.
+ if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v16f32, V1, V2, Mask,
+ Zeroable, Subtarget, DAG))
+ return Rotate;
+
return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG);
}
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index 5a0a9b7c3c25c..6f165507bc825 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -5153,6 +5153,16 @@ define <8 x i64> @test_mask_valign_q(<8 x i64> %a, <8 x i64> %b, <8 x i64> %src,
declare <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64>, <8 x i64>, i32, <8 x i64>, i8)
+define <16 x i32> @test_valign_d(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_valign_d:
+; CHECK: ## %bb.0:
+; CHECK-NEXT: valignd $3, %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf3,0x7d,0x48,0x03,0xc1,0x03]
+; CHECK-NEXT: ## zmm0 = zmm1[3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0,1,2]
+; CHECK-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
+ %res = call <16 x i32> @llvm.x86.avx512.mask.valign.d.512(<16 x i32> %a, <16 x i32> %b, i32 3, <16 x i32> zeroinitializer, i16 -1)
+ ret <16 x i32> %res
+}
+
define <16 x i32> @test_maskz_valign_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) {
; X86-LABEL: test_maskz_valign_d:
; X86: ## %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
index f31e815e186c3..79643cdeebabc 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -566,6 +566,16 @@ define <16 x i32> @shuffle_v16i32_16_zz_17_zz_18_zz_19_zz_20_zz_21_zz_22_zz_23_z
ret <16 x i32> %shuffle
}
+; Float-domain rotate matching __builtin_ia32_alignd512(A, B, 3) lowering.
+define <16 x float> @shuffle_v16f32_03_04_05_06_07_08_09_10_11_12_13_14_15_16_17_18(<16 x float> %a, <16 x float> %b) {
+; ALL-LABEL: shuffle_v16f32_03_04_05_06_07_08_09_10_11_12_13_14_15_16_17_18:
+; ALL: # %bb.0:
+; ALL-NEXT: valignd {{.*#+}} zmm0 = zmm1[3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0,1,2]
+; ALL-NEXT: retq
+ %shuffle = shufflevector <16 x float> %b, <16 x float> %a, <16 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>
+ ret <16 x float> %shuffle
+}
+
define <16 x i32> @shuffle_v16i32_01_02_03_04_05_06_07_08_09_10_11_12_13_14_15_16(<16 x i32> %a, <16 x i32> %b) {
; ALL-LABEL: shuffle_v16i32_01_02_03_04_05_06_07_08_09_10_11_12_13_14_15_16:
; ALL: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
index fce98cd470bcd..cbd72ef93897f 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
@@ -1485,6 +1485,16 @@ define <8 x double> @shuffle_v8f64_0zzzzzzz(<8 x double> %a) {
ret <8 x double> %shuffle
}
+; Float-domain rotate matching __builtin_ia32_alignq512(A, B, 2) lowering.
+define <8 x double> @shuffle_v8f64_23456789(<8 x double> %a, <8 x double> %b) {
+; ALL-LABEL: shuffle_v8f64_23456789:
+; ALL: # %bb.0:
+; ALL-NEXT: valignq {{.*#+}} zmm0 = zmm1[2,3,4,5,6,7],zmm0[0,1]
+; ALL-NEXT: ret{{[l|q]}}
+ %shuffle = shufflevector <8 x double> %b, <8 x double> %a, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+ ret <8 x double> %shuffle
+}
+
define <8 x i64> @shuffle_v8i64_12345678(<8 x i64> %a, <8 x i64> %b) {
; ALL-LABEL: shuffle_v8i64_12345678:
; ALL: # %bb.0:
More information about the llvm-commits
mailing list