[llvm] [X86] Enable i512 fshl/fshr lowering on avx512 targets (PR #185615)
Phoebe Wang via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 10 05:54:03 PDT 2026
================
@@ -5200,240 +4549,123 @@ define i512 @fshr_rot_i512_vector(<8 x i64> %v0, i512 %a2) nounwind {
;
; AVX512F-LABEL: fshr_rot_i512_vector:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %r15
-; AVX512F-NEXT: pushq %r14
-; AVX512F-NEXT: pushq %r13
-; AVX512F-NEXT: pushq %r12
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT: movq %rsi, %rcx
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512F-NEXT: vmovq %xmm2, %r8
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512F-NEXT: vpextrq $1, %xmm2, %r14
-; AVX512F-NEXT: vpextrq $1, %xmm3, %r11
-; AVX512F-NEXT: vmovq %xmm3, %rbx
-; AVX512F-NEXT: vmovq %xmm0, %rsi
-; AVX512F-NEXT: vmovq %xmm1, %r12
-; AVX512F-NEXT: testl $256, %ecx # imm = 0x100
-; AVX512F-NEXT: movq %r12, %r15
-; AVX512F-NEXT: cmoveq %rsi, %r15
-; AVX512F-NEXT: movq %r11, %r13
-; AVX512F-NEXT: cmoveq %r14, %r13
-; AVX512F-NEXT: movq %rbx, %r9
-; AVX512F-NEXT: cmoveq %r8, %r9
-; AVX512F-NEXT: movq %rax, %r10
-; AVX512F-NEXT: cmoveq %rdx, %r10
-; AVX512F-NEXT: cmoveq %r12, %rsi
-; AVX512F-NEXT: cmoveq %rbx, %r8
-; AVX512F-NEXT: cmoveq %rax, %rdx
-; AVX512F-NEXT: cmoveq %r11, %r14
-; AVX512F-NEXT: testb $-128, %cl
-; AVX512F-NEXT: movq %r10, %r11
-; AVX512F-NEXT: cmoveq %r14, %r11
-; AVX512F-NEXT: cmoveq %rdx, %r14
-; AVX512F-NEXT: movq %r9, %rbx
-; AVX512F-NEXT: cmoveq %rsi, %rbx
-; AVX512F-NEXT: cmoveq %r8, %rsi
-; AVX512F-NEXT: cmovneq %r13, %r10
-; AVX512F-NEXT: cmovneq %r15, %r9
-; AVX512F-NEXT: cmoveq %r13, %rdx
+; AVX512F-NEXT: andl $511, %esi # imm = 0x1FF
+; AVX512F-NEXT: vmovq %rsi, %xmm1
+; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm2
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [63,63]
+; AVX512F-NEXT: vpand %xmm1, %xmm2, %xmm3
+; AVX512F-NEXT: movl %esi, %eax
+; AVX512F-NEXT: shrl $6, %eax
+; AVX512F-NEXT: movl $-1, %ecx
+; AVX512F-NEXT: shlxl %eax, %ecx, %eax
+; AVX512F-NEXT: kmovw %eax, %k1
+; AVX512F-NEXT: vpcompressq %zmm0, %zmm4 {%k1} {z}
+; AVX512F-NEXT: vpsrlq %xmm3, %zmm4, %zmm3
+; AVX512F-NEXT: vpandn %xmm1, %xmm2, %xmm2
+; AVX512F-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT: valignq {{.*#+}} zmm4 = zmm4[1,2,3,4,5,6,7],zmm5[0]
+; AVX512F-NEXT: vpaddq %zmm4, %zmm4, %zmm4
+; AVX512F-NEXT: vpsllq %xmm2, %zmm4, %zmm2
; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: cmoveq %r15, %r8
-; AVX512F-NEXT: testb $64, %cl
-; AVX512F-NEXT: movq %r8, %rdi
-; AVX512F-NEXT: cmoveq %rdx, %rdi
-; AVX512F-NEXT: cmoveq %r9, %rdx
-; AVX512F-NEXT: cmoveq %r10, %r9
-; AVX512F-NEXT: cmoveq %rbx, %r10
-; AVX512F-NEXT: cmoveq %r11, %rbx
-; AVX512F-NEXT: cmoveq %rsi, %r11
-; AVX512F-NEXT: cmoveq %r14, %rsi
-; AVX512F-NEXT: cmovneq %r14, %r8
-; AVX512F-NEXT: movq %r8, %r14
-; AVX512F-NEXT: shrdq %cl, %rsi, %r14
-; AVX512F-NEXT: shrdq %cl, %r11, %rsi
-; AVX512F-NEXT: shrdq %cl, %rbx, %r11
-; AVX512F-NEXT: shrdq %cl, %r10, %rbx
-; AVX512F-NEXT: shrdq %cl, %r9, %r10
-; AVX512F-NEXT: shrdq %cl, %rdx, %r9
-; AVX512F-NEXT: shrdq %cl, %rdi, %rdx
-; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512F-NEXT: shrdq %cl, %r8, %rdi
-; AVX512F-NEXT: movq %rdi, 56(%rax)
-; AVX512F-NEXT: movq %rdx, 48(%rax)
-; AVX512F-NEXT: movq %r9, 40(%rax)
-; AVX512F-NEXT: movq %r10, 32(%rax)
-; AVX512F-NEXT: movq %rbx, 24(%rax)
-; AVX512F-NEXT: movq %r11, 16(%rax)
-; AVX512F-NEXT: movq %rsi, 8(%rax)
-; AVX512F-NEXT: movq %r14, (%rax)
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %r12
-; AVX512F-NEXT: popq %r13
-; AVX512F-NEXT: popq %r14
-; AVX512F-NEXT: popq %r15
+; AVX512F-NEXT: vporq %zmm3, %zmm2, %zmm2
+; AVX512F-NEXT: movq %rsi, %rdx
+; AVX512F-NEXT: xorq $511, %rdx # imm = 0x1FF
+; AVX512F-NEXT: vmovq %rdx, %xmm3
+; AVX512F-NEXT: vpbroadcastq %xmm3, %xmm3
+; AVX512F-NEXT: vpand %xmm1, %xmm3, %xmm4
+; AVX512F-NEXT: shrl $6, %edx
+; AVX512F-NEXT: shlxl %edx, %ecx, %ecx
+; AVX512F-NEXT: kmovw %ecx, %k1
+; AVX512F-NEXT: vpexpandq %zmm0, %zmm6 {%k1} {z}
+; AVX512F-NEXT: vpsllq %xmm4, %zmm6, %zmm4
+; AVX512F-NEXT: vpandn %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT: valignq {{.*#+}} zmm3 = zmm5[7],zmm6[0,1,2,3,4,5,6]
+; AVX512F-NEXT: vpsrlq $1, %zmm3, %zmm3
+; AVX512F-NEXT: vpsrlq %xmm1, %zmm3, %zmm1
+; AVX512F-NEXT: xorl %ecx, %ecx
+; AVX512F-NEXT: negq %rsi
+; AVX512F-NEXT: sbbl %ecx, %ecx
+; AVX512F-NEXT: vporq %zmm1, %zmm4, %zmm1
+; AVX512F-NEXT: kmovw %ecx, %k1
+; AVX512F-NEXT: vporq %zmm2, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT: vmovdqu64 %zmm0, (%rdi)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: fshr_rot_i512_vector:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %r15
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %r13
-; AVX512VL-NEXT: pushq %r12
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r11
-; AVX512VL-NEXT: movq %rsi, %rcx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512VL-NEXT: vmovq %xmm2, %r8
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512VL-NEXT: vpextrq $1, %xmm2, %rax
-; AVX512VL-NEXT: vmovq %xmm3, %rbx
-; AVX512VL-NEXT: vpextrq $1, %xmm3, %r15
-; AVX512VL-NEXT: vmovq %xmm0, %rsi
-; AVX512VL-NEXT: vmovq %xmm1, %r12
-; AVX512VL-NEXT: testl $256, %ecx # imm = 0x100
-; AVX512VL-NEXT: movq %r12, %r14
-; AVX512VL-NEXT: cmoveq %rsi, %r14
-; AVX512VL-NEXT: movq %r15, %r13
-; AVX512VL-NEXT: cmoveq %rax, %r13
-; AVX512VL-NEXT: movq %rbx, %r9
-; AVX512VL-NEXT: cmoveq %r8, %r9
-; AVX512VL-NEXT: movq %r11, %r10
-; AVX512VL-NEXT: cmoveq %rdx, %r10
-; AVX512VL-NEXT: cmoveq %r12, %rsi
-; AVX512VL-NEXT: cmoveq %rbx, %r8
-; AVX512VL-NEXT: cmoveq %r11, %rdx
-; AVX512VL-NEXT: cmoveq %r15, %rax
-; AVX512VL-NEXT: testb $-128, %cl
-; AVX512VL-NEXT: movq %r10, %r11
-; AVX512VL-NEXT: cmoveq %rax, %r11
-; AVX512VL-NEXT: cmoveq %rdx, %rax
-; AVX512VL-NEXT: movq %r9, %rbx
-; AVX512VL-NEXT: cmoveq %rsi, %rbx
-; AVX512VL-NEXT: cmoveq %r8, %rsi
-; AVX512VL-NEXT: cmovneq %r13, %r10
-; AVX512VL-NEXT: cmovneq %r14, %r9
-; AVX512VL-NEXT: cmoveq %r13, %rdx
-; AVX512VL-NEXT: cmoveq %r14, %r8
-; AVX512VL-NEXT: testb $64, %cl
-; AVX512VL-NEXT: movq %r8, %r14
-; AVX512VL-NEXT: cmoveq %rdx, %r14
-; AVX512VL-NEXT: cmoveq %r9, %rdx
-; AVX512VL-NEXT: cmoveq %r10, %r9
-; AVX512VL-NEXT: cmoveq %rbx, %r10
-; AVX512VL-NEXT: cmoveq %r11, %rbx
-; AVX512VL-NEXT: cmoveq %rsi, %r11
-; AVX512VL-NEXT: cmoveq %rax, %rsi
-; AVX512VL-NEXT: cmovneq %rax, %r8
-; AVX512VL-NEXT: movq %r8, %r15
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r15
-; AVX512VL-NEXT: shrdq %cl, %r11, %rsi
-; AVX512VL-NEXT: shrdq %cl, %rbx, %r11
-; AVX512VL-NEXT: shrdq %cl, %r10, %rbx
-; AVX512VL-NEXT: shrdq %cl, %r9, %r10
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r9
; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shrdq %cl, %r14, %rdx
-; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512VL-NEXT: shrdq %cl, %r8, %r14
-; AVX512VL-NEXT: movq %r14, 56(%rdi)
-; AVX512VL-NEXT: movq %rdx, 48(%rdi)
-; AVX512VL-NEXT: movq %r9, 40(%rdi)
-; AVX512VL-NEXT: movq %r10, 32(%rdi)
-; AVX512VL-NEXT: movq %rbx, 24(%rdi)
-; AVX512VL-NEXT: movq %r11, 16(%rdi)
-; AVX512VL-NEXT: movq %rsi, 8(%rdi)
-; AVX512VL-NEXT: movq %r15, (%rdi)
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r12
-; AVX512VL-NEXT: popq %r13
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %r15
+; AVX512VL-NEXT: andl $511, %esi # imm = 0x1FF
+; AVX512VL-NEXT: vpbroadcastq %rsi, %xmm1
+; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [63,63]
+; AVX512VL-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512VL-NEXT: movl %esi, %ecx
+; AVX512VL-NEXT: shrl $6, %ecx
+; AVX512VL-NEXT: movl $-1, %edx
+; AVX512VL-NEXT: shlxl %ecx, %edx, %ecx
+; AVX512VL-NEXT: kmovd %ecx, %k1
+; AVX512VL-NEXT: vpcompressq %zmm0, %zmm4 {%k1} {z}
+; AVX512VL-NEXT: vpsrlq %xmm3, %zmm4, %zmm3
+; AVX512VL-NEXT: vpandn %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT: valignq {{.*#+}} zmm4 = zmm4[1,2,3,4,5,6,7],zmm5[0]
+; AVX512VL-NEXT: vpaddq %zmm4, %zmm4, %zmm4
+; AVX512VL-NEXT: vpsllq %xmm1, %zmm4, %zmm1
+; AVX512VL-NEXT: vporq %zmm3, %zmm1, %zmm1
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: xorq $511, %rcx # imm = 0x1FF
+; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm3
+; AVX512VL-NEXT: vpand %xmm2, %xmm3, %xmm4
+; AVX512VL-NEXT: shrl $6, %ecx
+; AVX512VL-NEXT: shlxl %ecx, %edx, %ecx
+; AVX512VL-NEXT: kmovd %ecx, %k1
+; AVX512VL-NEXT: vpexpandq %zmm0, %zmm6 {%k1} {z}
+; AVX512VL-NEXT: vpsllq %xmm4, %zmm6, %zmm4
+; AVX512VL-NEXT: vpandn %xmm2, %xmm3, %xmm2
+; AVX512VL-NEXT: valignq {{.*#+}} zmm3 = zmm5[7],zmm6[0,1,2,3,4,5,6]
+; AVX512VL-NEXT: vpsrlq $1, %zmm3, %zmm3
+; AVX512VL-NEXT: vpsrlq %xmm2, %zmm3, %zmm2
+; AVX512VL-NEXT: vporq %zmm2, %zmm4, %zmm2
+; AVX512VL-NEXT: xorl %ecx, %ecx
+; AVX512VL-NEXT: negq %rsi
+; AVX512VL-NEXT: sbbl %ecx, %ecx
+; AVX512VL-NEXT: kmovd %ecx, %k1
+; AVX512VL-NEXT: vporq %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, 32(%rdi)
+; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: fshr_rot_i512_vector:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: pushq %r15
-; AVX512VBMI-NEXT: pushq %r14
-; AVX512VBMI-NEXT: pushq %r13
-; AVX512VBMI-NEXT: pushq %r12
-; AVX512VBMI-NEXT: pushq %rbx
-; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r11
-; AVX512VBMI-NEXT: movq %rsi, %rcx
-; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512VBMI-NEXT: vmovq %xmm2, %r8
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512VBMI-NEXT: vpextrq $1, %xmm2, %rax
-; AVX512VBMI-NEXT: vmovq %xmm3, %rbx
-; AVX512VBMI-NEXT: vpextrq $1, %xmm3, %r15
-; AVX512VBMI-NEXT: vmovq %xmm0, %rsi
-; AVX512VBMI-NEXT: vmovq %xmm1, %r12
-; AVX512VBMI-NEXT: testl $256, %ecx # imm = 0x100
-; AVX512VBMI-NEXT: movq %r12, %r14
-; AVX512VBMI-NEXT: cmoveq %rsi, %r14
-; AVX512VBMI-NEXT: movq %r15, %r13
-; AVX512VBMI-NEXT: cmoveq %rax, %r13
-; AVX512VBMI-NEXT: movq %rbx, %r9
-; AVX512VBMI-NEXT: cmoveq %r8, %r9
-; AVX512VBMI-NEXT: movq %r11, %r10
-; AVX512VBMI-NEXT: cmoveq %rdx, %r10
-; AVX512VBMI-NEXT: cmoveq %r12, %rsi
-; AVX512VBMI-NEXT: cmoveq %rbx, %r8
-; AVX512VBMI-NEXT: cmoveq %r11, %rdx
-; AVX512VBMI-NEXT: cmoveq %r15, %rax
-; AVX512VBMI-NEXT: testb $-128, %cl
-; AVX512VBMI-NEXT: movq %r10, %r11
-; AVX512VBMI-NEXT: cmoveq %rax, %r11
-; AVX512VBMI-NEXT: cmoveq %rdx, %rax
-; AVX512VBMI-NEXT: movq %r9, %rbx
-; AVX512VBMI-NEXT: cmoveq %rsi, %rbx
-; AVX512VBMI-NEXT: cmoveq %r8, %rsi
-; AVX512VBMI-NEXT: cmovneq %r13, %r10
-; AVX512VBMI-NEXT: cmovneq %r14, %r9
-; AVX512VBMI-NEXT: cmoveq %r13, %rdx
-; AVX512VBMI-NEXT: cmoveq %r14, %r8
-; AVX512VBMI-NEXT: testb $64, %cl
-; AVX512VBMI-NEXT: movq %r8, %r14
-; AVX512VBMI-NEXT: cmoveq %rdx, %r14
-; AVX512VBMI-NEXT: cmoveq %r9, %rdx
-; AVX512VBMI-NEXT: cmoveq %r10, %r9
-; AVX512VBMI-NEXT: cmoveq %rbx, %r10
-; AVX512VBMI-NEXT: cmoveq %r11, %rbx
-; AVX512VBMI-NEXT: cmoveq %rsi, %r11
-; AVX512VBMI-NEXT: cmoveq %rax, %rsi
-; AVX512VBMI-NEXT: cmovneq %rax, %r8
-; AVX512VBMI-NEXT: movq %r8, %r15
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r15
-; AVX512VBMI-NEXT: shrdq %cl, %r11, %rsi
-; AVX512VBMI-NEXT: shrdq %cl, %rbx, %r11
-; AVX512VBMI-NEXT: shrdq %cl, %r10, %rbx
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r9
; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shrdq %cl, %r14, %rdx
-; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512VBMI-NEXT: shrdq %cl, %r8, %r14
-; AVX512VBMI-NEXT: movq %r14, 56(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 48(%rdi)
-; AVX512VBMI-NEXT: movq %r9, 40(%rdi)
-; AVX512VBMI-NEXT: movq %r10, 32(%rdi)
-; AVX512VBMI-NEXT: movq %rbx, 24(%rdi)
-; AVX512VBMI-NEXT: movq %r11, 16(%rdi)
-; AVX512VBMI-NEXT: movq %rsi, 8(%rdi)
-; AVX512VBMI-NEXT: movq %r15, (%rdi)
-; AVX512VBMI-NEXT: popq %rbx
-; AVX512VBMI-NEXT: popq %r12
-; AVX512VBMI-NEXT: popq %r13
-; AVX512VBMI-NEXT: popq %r14
-; AVX512VBMI-NEXT: popq %r15
+; AVX512VBMI-NEXT: andl $511, %esi # imm = 0x1FF
+; AVX512VBMI-NEXT: movl %esi, %ecx
+; AVX512VBMI-NEXT: shrl $6, %ecx
+; AVX512VBMI-NEXT: movl $-1, %edx
+; AVX512VBMI-NEXT: shlxl %ecx, %edx, %ecx
+; AVX512VBMI-NEXT: kmovd %ecx, %k1
+; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm1 {%k1} {z}
+; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm3 = zmm1[1,2,3,4,5,6,7],zmm2[0]
+; AVX512VBMI-NEXT: vpbroadcastq %rsi, %zmm4
+; AVX512VBMI-NEXT: vpshrdvq %zmm4, %zmm3, %zmm1
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: xorq $511, %rcx # imm = 0x1FF
+; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm3
+; AVX512VBMI-NEXT: # kill: def $ecx killed $ecx killed $rcx
+; AVX512VBMI-NEXT: shrl $6, %ecx
+; AVX512VBMI-NEXT: shlxl %ecx, %edx, %ecx
+; AVX512VBMI-NEXT: kmovd %ecx, %k1
+; AVX512VBMI-NEXT: vpexpandq %zmm0, %zmm4 {%k1} {z}
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm2[7],zmm4[0,1,2,3,4,5,6]
+; AVX512VBMI-NEXT: vpshldvq %zmm3, %zmm2, %zmm4
+; AVX512VBMI-NEXT: xorl %ecx, %ecx
+; AVX512VBMI-NEXT: negq %rsi
+; AVX512VBMI-NEXT: sbbl %ecx, %ecx
+; AVX512VBMI-NEXT: kmovd %ecx, %k1
+; AVX512VBMI-NEXT: vporq %zmm1, %zmm4, %zmm0 {%k1}
----------------
phoebewang wrote:
I think a branch is better than unconditionally execute a lot of code But shift by 0 should be rare, so it doesn't matter in real
https://github.com/llvm/llvm-project/pull/185615
More information about the llvm-commits
mailing list