[llvm] ad810c1 - [X86] Improve PDEP/PEXT instructions scheduling on znver1/znver2 models (#217304)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 02:34:36 PDT 2026
Author: Simon Pilgrim
Date: 2026-08-27T09:34:23Z
New Revision: ad810c186e9059e554c179bab9d17527cd8f0247
URL: https://github.com/llvm/llvm-project/commit/ad810c186e9059e554c179bab9d17527cd8f0247
DIFF: https://github.com/llvm/llvm-project/commit/ad810c186e9059e554c179bab9d17527cd8f0247.diff
LOG: [X86] Improve PDEP/PEXT instructions scheduling on znver1/znver2 models (#217304)
znver1/znver2 have notoriously bad implementations of the PDEP/PEXT
scalar instructions, just using the WriteMicrocoded tag isn't useful as
it can't give any indication of the awful effect on ALU pipes or uops
flooding.
The measurements on uops.info/Agner are all over the place (and are very
input dependent), so I've taken an average of the worse case to at least
give the scheduler something to work with.
We've ignored these so far, but now we're close to generating
llvm.pdep/pext intrinsics (and scalarizing vector implementations).
Added:
Modified:
llvm/lib/Target/X86/X86ScheduleZnver1.td
llvm/lib/Target/X86/X86ScheduleZnver2.td
llvm/test/tools/llvm-mca/X86/Znver1/resources-bmi2.s
llvm/test/tools/llvm-mca/X86/Znver2/resources-bmi2.s
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver1.td b/llvm/lib/Target/X86/X86ScheduleZnver1.td
index b7f14ccbacf93..7f6c743c2a5c8 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver1.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver1.td
@@ -706,9 +706,34 @@ def : SchedAlias<WriteBitTestSetRegRMW, ZnWriteBTRSCm>;
// PDEP PEXT.
// r,r,r.
-def : InstRW<[WriteMicrocoded], (instregex "PDEP(32|64)rr", "PEXT(32|64)rr")>;
// r,r,m.
-def : InstRW<[WriteMicrocoded], (instregex "PDEP(32|64)rm", "PEXT(32|64)rm")>;
+def ZnWritePDEP32rr : SchedWriteRes<[ZnALU, ZnMultiplier]> {
+ let Latency = 53;
+ let NumMicroOps = 95;
+ let ReleaseAtCycles = [1, 53];
+}
+def : InstRW<[ZnWritePDEP32rr], (instrs PDEP32rr, PEXT32rr)>;
+
+def ZnWritePDEP32rm : SchedWriteRes<[ZnALU, ZnMultiplier, ZnAGU]> {
+ let Latency = 56;
+ let NumMicroOps = 96;
+ let ReleaseAtCycles = [1, 55, 1];
+}
+def : InstRW<[ZnWritePDEP32rm], (instrs PDEP32rm, PEXT32rm)>;
+
+def ZnWritePDEP64rr : SchedWriteRes<[ZnALU, ZnMultiplier]> {
+ let Latency = 86;
+ let NumMicroOps = 165;
+ let ReleaseAtCycles = [1, 86];
+}
+def : InstRW<[ZnWritePDEP64rr], (instrs PDEP64rr, PEXT64rr)>;
+
+def ZnWritePDEP64rm : SchedWriteRes<[ZnALU, ZnMultiplier, ZnAGU]> {
+ let Latency = 89;
+ let NumMicroOps = 166;
+ let ReleaseAtCycles = [1, 89, 1];
+}
+def : InstRW<[ZnWritePDEP64rm], (instrs PDEP64rm, PEXT64rm)>;
// RCR RCL.
// m,i.
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver2.td b/llvm/lib/Target/X86/X86ScheduleZnver2.td
index a981a5eb3f1c1..cfcc496fa6f31 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver2.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver2.td
@@ -716,9 +716,34 @@ def : SchedAlias<WriteBitTestSetRegRMW, Zn2WriteBTRSCm>;
// PDEP PEXT.
// r,r,r.
-def : InstRW<[WriteMicrocoded], (instregex "PDEP(32|64)rr", "PEXT(32|64)rr")>;
// r,r,m.
-def : InstRW<[WriteMicrocoded], (instregex "PDEP(32|64)rm", "PEXT(32|64)rm")>;
+def Zn2WritePDEP32rr : SchedWriteRes<[Zn2ALU, Zn2Multiplier]> {
+ let Latency = 53;
+ let NumMicroOps = 95;
+ let ReleaseAtCycles = [1, 53];
+}
+def : InstRW<[Zn2WritePDEP32rr], (instrs PDEP32rr, PEXT32rr)>;
+
+def Zn2WritePDEP32rm : SchedWriteRes<[Zn2ALU, Zn2Multiplier, Zn2AGU]> {
+ let Latency = 56;
+ let NumMicroOps = 96;
+ let ReleaseAtCycles = [1, 55, 1];
+}
+def : InstRW<[Zn2WritePDEP32rm], (instrs PDEP32rm, PEXT32rm)>;
+
+def Zn2WritePDEP64rr : SchedWriteRes<[Zn2ALU, Zn2Multiplier]> {
+ let Latency = 86;
+ let NumMicroOps = 165;
+ let ReleaseAtCycles = [1, 86];
+}
+def : InstRW<[Zn2WritePDEP64rr], (instrs PDEP64rr, PEXT64rr)>;
+
+def Zn2WritePDEP64rm : SchedWriteRes<[Zn2ALU, Zn2Multiplier, Zn2AGU]> {
+ let Latency = 89;
+ let NumMicroOps = 166;
+ let ReleaseAtCycles = [1, 89, 1];
+}
+def : InstRW<[Zn2WritePDEP64rm], (instrs PDEP64rm, PEXT64rm)>;
// RCR RCL.
// m,i.
diff --git a/llvm/test/tools/llvm-mca/X86/Znver1/resources-bmi2.s b/llvm/test/tools/llvm-mca/X86/Znver1/resources-bmi2.s
index 32785f92675d4..17cac2bac2267 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver1/resources-bmi2.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver1/resources-bmi2.s
@@ -66,14 +66,14 @@ shrx %rax, (%rbx), %rcx
# CHECK-NEXT: 1 8 2.00 * mulxl (%rax), %ebx, %ecx
# CHECK-NEXT: 1 3 2.00 mulxq %rax, %rbx, %rcx
# CHECK-NEXT: 1 8 2.00 * mulxq (%rax), %rbx, %rcx
-# CHECK-NEXT: 1 100 0.25 pdepl %eax, %ebx, %ecx
-# CHECK-NEXT: 1 100 0.25 * pdepl (%rax), %ebx, %ecx
-# CHECK-NEXT: 1 100 0.25 pdepq %rax, %rbx, %rcx
-# CHECK-NEXT: 1 100 0.25 * pdepq (%rax), %rbx, %rcx
-# CHECK-NEXT: 1 100 0.25 pextl %eax, %ebx, %ecx
-# CHECK-NEXT: 1 100 0.25 * pextl (%rax), %ebx, %ecx
-# CHECK-NEXT: 1 100 0.25 pextq %rax, %rbx, %rcx
-# CHECK-NEXT: 1 100 0.25 * pextq (%rax), %rbx, %rcx
+# CHECK-NEXT: 95 53 53.00 pdepl %eax, %ebx, %ecx
+# CHECK-NEXT: 96 56 55.00 * pdepl (%rax), %ebx, %ecx
+# CHECK-NEXT: 165 86 86.00 pdepq %rax, %rbx, %rcx
+# CHECK-NEXT: 166 89 89.00 * pdepq (%rax), %rbx, %rcx
+# CHECK-NEXT: 95 53 53.00 pextl %eax, %ebx, %ecx
+# CHECK-NEXT: 96 56 55.00 * pextl (%rax), %ebx, %ecx
+# CHECK-NEXT: 165 86 86.00 pextq %rax, %rbx, %rcx
+# CHECK-NEXT: 166 89 89.00 * pextq (%rax), %rbx, %rcx
# CHECK-NEXT: 1 1 0.25 rorxl $1, %eax, %ecx
# CHECK-NEXT: 2 5 0.50 * rorxl $1, (%rax), %ecx
# CHECK-NEXT: 1 1 0.25 rorxq $1, %rax, %rcx
@@ -107,7 +107,7 @@ shrx %rax, (%rbx), %rcx
# CHECK: Resource pressure per iteration:
# CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11]
-# CHECK-NEXT: 6.00 6.00 5.00 9.00 5.00 5.00 - - - - - 8.00
+# CHECK-NEXT: 8.00 8.00 7.00 11.00 7.00 7.00 - - - - - 574.00
# CHECK: Resource pressure by instruction:
# CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions:
@@ -119,14 +119,14 @@ shrx %rax, (%rbx), %rcx
# CHECK-NEXT: 0.50 0.50 - 1.00 - - - - - - - 2.00 mulxl (%rax), %ebx, %ecx
# CHECK-NEXT: - - - 1.00 - - - - - - - 2.00 mulxq %rax, %rbx, %rcx
# CHECK-NEXT: 0.50 0.50 - 1.00 - - - - - - - 2.00 mulxq (%rax), %rbx, %rcx
-# CHECK-NEXT: - - - - - - - - - - - - pdepl %eax, %ebx, %ecx
-# CHECK-NEXT: - - - - - - - - - - - - pdepl (%rax), %ebx, %ecx
-# CHECK-NEXT: - - - - - - - - - - - - pdepq %rax, %rbx, %rcx
-# CHECK-NEXT: - - - - - - - - - - - - pdepq (%rax), %rbx, %rcx
-# CHECK-NEXT: - - - - - - - - - - - - pextl %eax, %ebx, %ecx
-# CHECK-NEXT: - - - - - - - - - - - - pextl (%rax), %ebx, %ecx
-# CHECK-NEXT: - - - - - - - - - - - - pextq %rax, %rbx, %rcx
-# CHECK-NEXT: - - - - - - - - - - - - pextq (%rax), %rbx, %rcx
+# CHECK-NEXT: - - 0.25 0.25 0.25 0.25 - - - - - 53.00 pdepl %eax, %ebx, %ecx
+# CHECK-NEXT: 0.50 0.50 0.25 0.25 0.25 0.25 - - - - - 55.00 pdepl (%rax), %ebx, %ecx
+# CHECK-NEXT: - - 0.25 0.25 0.25 0.25 - - - - - 86.00 pdepq %rax, %rbx, %rcx
+# CHECK-NEXT: 0.50 0.50 0.25 0.25 0.25 0.25 - - - - - 89.00 pdepq (%rax), %rbx, %rcx
+# CHECK-NEXT: - - 0.25 0.25 0.25 0.25 - - - - - 53.00 pextl %eax, %ebx, %ecx
+# CHECK-NEXT: 0.50 0.50 0.25 0.25 0.25 0.25 - - - - - 55.00 pextl (%rax), %ebx, %ecx
+# CHECK-NEXT: - - 0.25 0.25 0.25 0.25 - - - - - 86.00 pextq %rax, %rbx, %rcx
+# CHECK-NEXT: 0.50 0.50 0.25 0.25 0.25 0.25 - - - - - 89.00 pextq (%rax), %rbx, %rcx
# CHECK-NEXT: - - 0.25 0.25 0.25 0.25 - - - - - - rorxl $1, %eax, %ecx
# CHECK-NEXT: 0.50 0.50 0.25 0.25 0.25 0.25 - - - - - - rorxl $1, (%rax), %ecx
# CHECK-NEXT: - - 0.25 0.25 0.25 0.25 - - - - - - rorxq $1, %rax, %rcx
diff --git a/llvm/test/tools/llvm-mca/X86/Znver2/resources-bmi2.s b/llvm/test/tools/llvm-mca/X86/Znver2/resources-bmi2.s
index 1562e859a6b2a..959f8128dc698 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver2/resources-bmi2.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver2/resources-bmi2.s
@@ -66,14 +66,14 @@ shrx %rax, (%rbx), %rcx
# CHECK-NEXT: 1 7 2.00 * mulxl (%rax), %ebx, %ecx
# CHECK-NEXT: 1 3 2.00 mulxq %rax, %rbx, %rcx
# CHECK-NEXT: 1 7 2.00 * mulxq (%rax), %rbx, %rcx
-# CHECK-NEXT: 1 100 0.25 pdepl %eax, %ebx, %ecx
-# CHECK-NEXT: 1 100 0.25 * pdepl (%rax), %ebx, %ecx
-# CHECK-NEXT: 1 100 0.25 pdepq %rax, %rbx, %rcx
-# CHECK-NEXT: 1 100 0.25 * pdepq (%rax), %rbx, %rcx
-# CHECK-NEXT: 1 100 0.25 pextl %eax, %ebx, %ecx
-# CHECK-NEXT: 1 100 0.25 * pextl (%rax), %ebx, %ecx
-# CHECK-NEXT: 1 100 0.25 pextq %rax, %rbx, %rcx
-# CHECK-NEXT: 1 100 0.25 * pextq (%rax), %rbx, %rcx
+# CHECK-NEXT: 95 53 53.00 pdepl %eax, %ebx, %ecx
+# CHECK-NEXT: 96 56 55.00 * pdepl (%rax), %ebx, %ecx
+# CHECK-NEXT: 165 86 86.00 pdepq %rax, %rbx, %rcx
+# CHECK-NEXT: 166 89 89.00 * pdepq (%rax), %rbx, %rcx
+# CHECK-NEXT: 95 53 53.00 pextl %eax, %ebx, %ecx
+# CHECK-NEXT: 96 56 55.00 * pextl (%rax), %ebx, %ecx
+# CHECK-NEXT: 165 86 86.00 pextq %rax, %rbx, %rcx
+# CHECK-NEXT: 166 89 89.00 * pextq (%rax), %rbx, %rcx
# CHECK-NEXT: 1 1 0.25 rorxl $1, %eax, %ecx
# CHECK-NEXT: 2 5 0.33 * rorxl $1, (%rax), %ecx
# CHECK-NEXT: 1 1 0.25 rorxq $1, %rax, %rcx
@@ -108,7 +108,7 @@ shrx %rax, (%rbx), %rcx
# CHECK: Resource pressure per iteration:
# CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] [12]
-# CHECK-NEXT: 4.00 4.00 4.00 5.00 9.00 5.00 5.00 - - - - - 8.00
+# CHECK-NEXT: 5.33 5.33 5.33 7.00 11.00 7.00 7.00 - - - - - 574.00
# CHECK: Resource pressure by instruction:
# CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] [12] Instructions:
@@ -120,14 +120,14 @@ shrx %rax, (%rbx), %rcx
# CHECK-NEXT: 0.33 0.33 0.33 - 1.00 - - - - - - - 2.00 mulxl (%rax), %ebx, %ecx
# CHECK-NEXT: - - - - 1.00 - - - - - - - 2.00 mulxq %rax, %rbx, %rcx
# CHECK-NEXT: 0.33 0.33 0.33 - 1.00 - - - - - - - 2.00 mulxq (%rax), %rbx, %rcx
-# CHECK-NEXT: - - - - - - - - - - - - - pdepl %eax, %ebx, %ecx
-# CHECK-NEXT: - - - - - - - - - - - - - pdepl (%rax), %ebx, %ecx
-# CHECK-NEXT: - - - - - - - - - - - - - pdepq %rax, %rbx, %rcx
-# CHECK-NEXT: - - - - - - - - - - - - - pdepq (%rax), %rbx, %rcx
-# CHECK-NEXT: - - - - - - - - - - - - - pextl %eax, %ebx, %ecx
-# CHECK-NEXT: - - - - - - - - - - - - - pextl (%rax), %ebx, %ecx
-# CHECK-NEXT: - - - - - - - - - - - - - pextq %rax, %rbx, %rcx
-# CHECK-NEXT: - - - - - - - - - - - - - pextq (%rax), %rbx, %rcx
+# CHECK-NEXT: - - - 0.25 0.25 0.25 0.25 - - - - - 53.00 pdepl %eax, %ebx, %ecx
+# CHECK-NEXT: 0.33 0.33 0.33 0.25 0.25 0.25 0.25 - - - - - 55.00 pdepl (%rax), %ebx, %ecx
+# CHECK-NEXT: - - - 0.25 0.25 0.25 0.25 - - - - - 86.00 pdepq %rax, %rbx, %rcx
+# CHECK-NEXT: 0.33 0.33 0.33 0.25 0.25 0.25 0.25 - - - - - 89.00 pdepq (%rax), %rbx, %rcx
+# CHECK-NEXT: - - - 0.25 0.25 0.25 0.25 - - - - - 53.00 pextl %eax, %ebx, %ecx
+# CHECK-NEXT: 0.33 0.33 0.33 0.25 0.25 0.25 0.25 - - - - - 55.00 pextl (%rax), %ebx, %ecx
+# CHECK-NEXT: - - - 0.25 0.25 0.25 0.25 - - - - - 86.00 pextq %rax, %rbx, %rcx
+# CHECK-NEXT: 0.33 0.33 0.33 0.25 0.25 0.25 0.25 - - - - - 89.00 pextq (%rax), %rbx, %rcx
# CHECK-NEXT: - - - 0.25 0.25 0.25 0.25 - - - - - - rorxl $1, %eax, %ecx
# CHECK-NEXT: 0.33 0.33 0.33 0.25 0.25 0.25 0.25 - - - - - - rorxl $1, (%rax), %ecx
# CHECK-NEXT: - - - 0.25 0.25 0.25 0.25 - - - - - - rorxq $1, %rax, %rcx
More information about the llvm-commits
mailing list