[libcxx-commits] [libcxx] [libc++] Add an ABI flag to optimize mersenne_twister_engine (PR #206423)
via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Jul 1 01:07:34 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-libcxx
Author: Nikolas Klauser (philnik777)
<details>
<summary>Changes</summary>
Instead of updating the status array every time `operator()` is called, we can instead update it once every time the entire array has been read. That allows the compile to vectorize the updating code, improving performance.
Apple M4:
```
Benchmark old new Difference % Difference
--------------------------- -------- ----- ------------ --------------
std::mt19937::operator() 1.39 0.65 -0.74 -53.21%
std::mt19937_64::operator() 1.46 1.00 -0.46 -31.72%
```
Fixes #<!-- -->197221
---
Patch is 51.61 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/206423.diff
5 Files Affected:
- (modified) libcxx/include/__configuration/abi.h (+1)
- (modified) libcxx/include/__random/mersenne_twister_engine.h (+36-7)
- (modified) libcxx/test/benchmarks/random.bench.cpp (+11)
- (modified) libcxx/test/std/numerics/rand/rand.eng/rand.eng.mers/ctor_result_type.pass.cpp (+160-29)
- (modified) libcxx/test/std/numerics/rand/rand.eng/rand.eng.mers/ctor_sseq.pass.cpp (+261-128)
``````````diff
diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h
index 51c82d99eec30..7a4c479fd1636 100644
--- a/libcxx/include/__configuration/abi.h
+++ b/libcxx/include/__configuration/abi.h
@@ -77,6 +77,7 @@
# define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
# define _LIBCPP_ABI_TRIVIALLY_COPYABLE_BIT_ITERATOR
# define _LIBCPP_ABI_USE_SMALL_DEQUE_BLOCK_SIZE
+# define _LIBCPP_ABI_VECTORIZED_MERSENNE_TWISTER_ENGINE
#elif _LIBCPP_ABI_VERSION == 1
// Feature macros for disabling pre ABI v1 features. All of these options
diff --git a/libcxx/include/__random/mersenne_twister_engine.h b/libcxx/include/__random/mersenne_twister_engine.h
index ce82b3387dbd2..f1d92a1ede932 100644
--- a/libcxx/include/__random/mersenne_twister_engine.h
+++ b/libcxx/include/__random/mersenne_twister_engine.h
@@ -181,6 +181,9 @@ class mersenne_twister_engine {
for (size_t __i = 1; __i < __n; ++__i)
__x_[__i] = (__f * (__x_[__i - 1] ^ __rshift<__w - 2>(__x_[__i - 1])) + __i) & _Max;
__i_ = 0;
+#ifdef _LIBCPP_ABI_VECTORIZED_MERSENNE_TWISTER_ENGINE
+ __update_all_states();
+#endif
}
template <class _Sseq, __enable_if_t<__is_seed_sequence_v<_Sseq, mersenne_twister_engine>, int> = 0>
_LIBCPP_HIDE_FROM_ABI void seed(_Sseq& __q) {
@@ -203,17 +206,43 @@ class mersenne_twister_engine {
return;
__x_[0] = result_type(1) << (__w - 1);
}
+#ifdef _LIBCPP_ABI_VECTORIZED_MERSENNE_TWISTER_ENGINE
+ __update_all_states();
+#endif
+ }
+
+ void __update_state(size_t __i, size_t __k) {
+ const size_t __j = (__i + 1) % __n;
+ const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
+ const result_type __yp = (__x_[__i] & ~__mask) | (__x_[__j] & __mask);
+ __x_[__i] = __x_[__k] ^ __rshift<1>(__yp) ^ (__a * (__yp & 1));
+ }
+
+ void __update_state(size_t __i) { __update_state(__i, (__i + __m) % __n); }
+
+ void __update_all_states() {
+ size_t __i = 0;
+ // This is split into two loops to help the compiler vectorize the code
+ for (; __i != (__n - __m); ++__i)
+ __update_state(__i);
+ for (size_t __j = 0; __i != __n; ++__i, ++__j)
+ __update_state(__i, __j);
}
// generating functions
[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI result_type operator()() {
- const size_t __j = (__i_ + 1) % __n;
- const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
- const result_type __yp = (__x_[__i_] & ~__mask) | (__x_[__j] & __mask);
- const size_t __k = (__i_ + __m) % __n;
- __x_[__i_] = __x_[__k] ^ __rshift<1>(__yp) ^ (__a * (__yp & 1));
- result_type __z = __x_[__i_] ^ (__rshift<__u>(__x_[__i_]) & __d);
- __i_ = __j;
+#ifdef _LIBCPP_ABI_VECTORIZED_MERSENNE_TWISTER_ENGINE
+ result_type __val = __x_[__i_];
+ if (++__i_ == __n) [[__unlikely__]] {
+ __update_all_states();
+ __i_ = 0;
+ }
+#else
+ __update_state(__i_);
+ result_type __val = __x_[__i_];
+ __i_ = (__i_ + 1) % __n;
+#endif
+ result_type __z = __val ^ (__rshift<__u>(__val) & __d);
__z ^= __lshift<__s>(__z) & __b;
__z ^= __lshift<__t>(__z) & __c;
return __z ^ __rshift<__l>(__z);
diff --git a/libcxx/test/benchmarks/random.bench.cpp b/libcxx/test/benchmarks/random.bench.cpp
index e6af4c3e26eaf..3f73504949f62 100644
--- a/libcxx/test/benchmarks/random.bench.cpp
+++ b/libcxx/test/benchmarks/random.bench.cpp
@@ -34,4 +34,15 @@ static void BM_SeedSeq_Generate(benchmark::State& state) {
}
BENCHMARK(BM_SeedSeq_Generate)->Ranges({{1, MAX_SEED_LEN}, {1, MAX_BUFFER_LEN}});
+template <class Engine>
+static void BM_engine(benchmark::State& state) {
+ Engine engine;
+
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(engine());
+ }
+}
+BENCHMARK(BM_engine<std::mt19937_64>)->Name("std::mt19937_64::operator()");
+BENCHMARK(BM_engine<std::mt19937>)->Name("std::mt19937::operator()");
+
BENCHMARK_MAIN();
diff --git a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.mers/ctor_result_type.pass.cpp b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.mers/ctor_result_type.pass.cpp
index 02df0345525cb..b261f37450a81 100644
--- a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.mers/ctor_result_type.pass.cpp
+++ b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.mers/ctor_result_type.pass.cpp
@@ -26,23 +26,21 @@
#include "test_macros.h"
#if TEST_STD_VER >= 11
-#include "make_implicit.h"
-#include "test_convertible.h"
+# include "make_implicit.h"
+# include "test_convertible.h"
#endif
template <class T>
-std::string
-to_string(T const &e)
-{
- std::ostringstream os;
- os << e;
- return os.str();
+std::string to_string(T const& e) {
+ std::ostringstream os;
+ os << e;
+ return os.str();
}
-void
-test1()
-{
- const char* a = "0 1 1812433255 1900727105 1208447044 2481403966 4042607538 337614300 "
+void test1() {
+ const char* a =
+#ifndef _LIBCPP_ABI_VECTORIZED_MERSENNE_TWISTER_ENGINE
+ "0 1 1812433255 1900727105 1208447044 2481403966 4042607538 337614300 "
"3232553940 1018809052 3202401494 1775180719 3192392114 594215549 184016991 "
"829906058 610491522 3879932251 3139825610 297902587 4075895579 2943625357 "
"3530655617 1423771745 2135928312 2891506774 1066338622 135451537 933040465 "
@@ -137,15 +135,82 @@ test1()
"2361884409 2860936803 683833250 3291277128 1686857206 1112632275 "
"1200680507 3342928196 2677058150 939442136 3407104669 2906783932 "
"3668048733 2030009470 1910839172 1234925283 3575831445 123595418 "
- "2362440495 3048484911 1796872496";
- std::mt19937 e1(0);
- assert(to_string(e1) == a);
+ "2362440495 3048484911 1796872496"
+#else
+ "2443250962 1093594115 1878467924 2709361018 1101979660 3904844661 676747479 2085143622 1056793272 3812477442 "
+ "2168787041 275552121 2696932952 3432054210 1657102335 3518946594 962584079 1051271004 3806145045 1414436097 "
+ "2032348584 1661738718 1116708477 2562755208 3176189976 696824676 2399811678 3992505346 569184356 2626558620 "
+ "136797809 4273176064 296167901 3430730584 331909803 1908676996 1950065095 604298543 3615988338 1570232852 "
+ "1028209748 1511467721 2411887154 4210753555 3096762720 423429618 659966766 2937509307 2222847265 378636552 "
+ "1142109618 2509241601 1521729757 888533219 250885260 2455816244 4046047811 1947467789 1395351953 2388948566 "
+ "934627940 194642258 1429256273 2139959677 1543740405 1569613451 4061840539 2075690423 824532376 844152077 "
+ "3218002536 897315311 823414659 1007534698 3139313867 2332794462 3675156583 722594429 3715123011 2656840676 "
+ "2373277715 1516144243 1538161999 1399719297 1448820876 3180842436 2020192709 3279498509 1048107540 3094290376 "
+ "3706920703 3810051683 4003871775 1069557309 151535004 667695756 861767313 1643171871 3649507728 3785763588 "
+ "737899497 2332908276 4082123854 1584539673 3929877280 1046086041 1304062606 2311324008 1190424879 538647603 "
+ "1728806718 148948582 659721465 2351006009 185647941 1825934330 2126619619 2230539295 3878421575 965937340 "
+ "2835563987 419858879 1527311104 2042333483 2745134265 3540517373 1100062165 2308613697 2690739856 2048293545 "
+ "2742876486 2408502115 53398423 3089771461 1398529613 2132477261 504134301 1535108421 1968507173 3412472770 "
+ "4048517196 3509143071 857063211 1429875570 4120479871 2786574360 2470799871 1644104875 2326863162 2033900387 "
+ "274888909 2629973384 1154405937 1839988197 2163830694 268733679 2928953965 387153042 3790900490 3202349717 "
+ "3799209681 2524871464 3801447311 3543081470 1801633977 3870370950 1581678156 358283485 4133385582 4052311650 "
+ "249498956 1712028623 3618329357 3395106946 2704820515 26653208 2437509555 4192186371 3639946838 2615917799 "
+ "294624699 463886267 2477203673 2514051526 4162882381 2937629358 3180524454 1388392394 2637132139 1999482082 "
+ "1870663339 2852309250 3489430700 1116063402 1673637188 1941955790 4081919937 2322920953 3500518887 2722575905 "
+ "405031829 3728964889 3464559963 665516555 1626835870 2523729468 238289561 497556937 2581429740 3456619478 "
+ "2601389837 820426192 268141532 2654829421 349692193 2426957585 1465734488 85095426 1080895536 2823366942 "
+ "379322515 2938986739 1561328720 2940081697 607513620 2724557130 2841377034 14123440 1085631349 1977859413 "
+ "1325586902 1690398943 2516154740 4144373551 555899232 1472707937 3318086852 719779833 3545481907 1384350857 "
+ "191709822 2924586997 3905397527 2528674757 2377804096 2742818576 4162971050 1916415976 2229076768 132706400 "
+ "1337415646 251604474 4168783505 819499074 1213291699 2098930479 1009044052 1060792037 2169940733 2973281031 "
+ "636656529 1946890813 367099469 4076624283 338339529 561343735 3309733580 3895891229 516195845 3075613489 "
+ "3076388119 4088758753 3895582245 1192445752 2923797073 2272605437 2964193665 943979628 3263904824 1852145108 "
+ "1911050628 1596880711 479795791 212699077 2172697748 1764175804 3951845064 2182587171 2396848133 2718837214 "
+ "2934177572 3638054152 914795167 1028977354 1744985339 1161407853 2069746822 3109934340 3666150054 729399111 "
+ "2911998815 152809615 417267263 1064343325 2751707923 1833270972 886939906 4054231201 1410366452 976142646 "
+ "1452795786 1904234879 4231894541 1505392175 3498767279 833704604 2100592828 688268099 1752675602 1975556664 "
+ "2102210733 698665709 1232104968 4051336323 1740516649 3131721670 3755143232 3433756601 138325540 1566724819 "
+ "3146312914 3262098346 1400757929 3821885720 356740291 4236900846 3962771215 3383537419 3432281817 3700484231 "
+ "1124512300 4040103150 2152814065 357241942 3507144572 3156778000 4029389048 1495464238 2710816120 618937484 "
+ "923913256 3233161411 3113338135 2380534800 2989134055 792458413 2073679910 2733527896 1078384210 2832175364 "
+ "501385130 1554930792 2339252603 535487294 2983664186 924228889 2127502206 2847205678 836935943 1902192892 "
+ "4094716961 3296730761 2307406435 281523156 728701372 3943302480 183005637 2660706047 2320033913 2011717602 "
+ "219970222 890345073 2479236311 653839655 38538414 3266308455 3726363269 1256762826 712114913 3499059198 "
+ "2847394904 3246075282 1299462737 830090526 369065487 3899451086 1825910875 2697294244 2553197317 2501134241 "
+ "930821716 2841281141 3920808923 3209045349 3381621736 2145077949 1958091404 371971179 3064957112 3151429184 "
+ "2833651995 1089438243 2146134351 1426568997 3528669710 1043897757 3052263589 3205925012 4076215371 2622476950 "
+ "2385717100 2179892838 3820135605 1574628644 177604240 1712062317 1192081671 1107476027 837376039 2818244460 "
+ "1382379731 2749207910 2644203474 1508860349 3006844699 3157252375 3789683778 1985542020 2309789126 1624089624 "
+ "3660683301 1432444043 1172735898 3271587635 841387671 3614623669 3002115850 1078807738 1095345284 1707086713 "
+ "4263485053 445506874 3148772337 2716917408 3836812085 1026228926 491069724 3983218285 2410582206 3821759029 "
+ "3789305873 1223256738 1677180485 1481771733 1670286271 2852111711 1628943665 2289772949 3483580787 2129860019 "
+ "3955695109 28974017 2289227173 1725243773 483357125 111587742 347631238 1805029491 839978009 339030914 "
+ "2442782449 1701726526 1481742442 3703534067 3796147861 235078546 3195450129 1420184214 998981944 163541155 "
+ "2087693215 3827554748 3003331872 1967967170 160190917 544473278 3884602428 4180513186 1603446876 2191744556 "
+ "1938251010 291242497 1764195107 4076051069 3860380798 806452940 2631256196 949838208 4096577802 1519225265 "
+ "2296800559 1015848400 2864792481 88004533 195132439 966918321 1451347413 2322994051 3585658769 1771598947 "
+ "3308395968 950148946 3846061568 564475171 1345892193 2439581032 2907206869 1297626422 3779454571 818255727 "
+ "705738065 3915384184 4133726754 2724859327 3773743817 326325901 2766390048 2562130418 1885210076 2622324981 "
+ "1727396790 1742653294 3133131942 3790857080 197042473 2868410679 1122976862 2264474389 3607393378 744386048 "
+ "3122128592 1101489146 4064675687 2818166901 2282076661 370818870 3053423257 2272652338 4077081984 4284213290 "
+ "1728304803 3902867896 1643530236 306212760 144705843 498541197 3103269575 4116304893 1646288275 2110757680 "
+ "4092919505 3390929578 1828116798 1847914177 1156754878 2985404552 3970881126 3035874635 2189824678 2637114092 "
+ "2588332748 3917818803 3302695646 1758418726 1934759438 3871576937 1360769189 1051808374 1881390541 757367126 "
+ "3253778407 1685049720 805311189 934898036 95682794 2052240235 3475933669 2213474191 3341273357 2443286051 "
+ "4272536794 696819230 878775027 3332634155 3405126958 2824772573 95284505 3040640920 2412859827 2959515765 "
+ "1764792340 1504924482 2116910326 2293943003 3199262907 1443740894 3900566588 343066614 1927475335 249220892 "
+ "2920636502 3593328327 2648828502 1678096082"
+#endif // _LIBCPP_ABI_VECTORIZED_MERSENNE_TWISTER_ENGINE
+ ;
+ std::mt19937 e1(0);
+ LIBCPP_ASSERT(to_string(e1) == a);
+ (void)a;
}
-void
-test2()
-{
- const char* a = "0 1 6364136223846793007 13885033948157127961 "
+void test2() {
+ const char* a =
+#ifndef _LIBCPP_ABI_VECTORIZED_MERSENNE_TWISTER_ENGINE
+ "0 1 6364136223846793007 13885033948157127961 "
"15324573939901584278 12737837167382305846 15195339788985155882 "
"6554113247712070460 17235932740818599105 13007415075556305955 "
"6585479514541334743 8274505865835507625 1718218088692873364 "
@@ -248,9 +313,76 @@ test2()
"14091690436120485477 15763282477731738396 16394237160547425954 "
"5066318118328746621 13140493775100916989 6371148952471982853 "
"15150289760867914983 4931341382074091848 12635920082410445322 "
- "8498109357807439006 14836776625250834986";
- std::mt19937_64 e1(0);
- assert(to_string(e1) == a);
+ "8498109357807439006 14836776625250834986"
+#else
+ "4640967165370225016 475580846709610041 992132072322742490 2879266425597847059 9859212410995834590 "
+ "13110412116137766394 2786888605138474240 12527462973726820895 5232191189150589434 8148228587345305383 "
+ "15985190208517110588 17155057006866468333 5040233137627218077 12716217438209898279 7628670991140263350 "
+ "9633431818170735083 8029874679197696152 2546188705250129595 13105544608186384869 6197138528956131705 "
+ "2500767142257833113 4057403796030124452 4801410225133580521 12712510024708253355 1478853541812684571 "
+ "16899998798183024166 763564441705181102 13204668179160915419 6975570844831914277 12029362619687152075 "
+ "17592155920476593974 1391138604394782251 13557582389860889375 6018426101890403504 7401881601334221625 "
+ "281124061034155254 2261645520472923927 10254468173793190351 4336071085409420785 7075410027066751424 "
+ "7438312290102887001 15139389032212795768 12949861808476919158 5221320262711473213 18444574859163569422 "
+ "3327484732560520915 10371833210785813703 494705948341138486 313693460050824934 801700386546917424 "
+ "3333797513889190239 11734932229126290067 4001689285737041461 17077402128101531244 17161815342609821634 "
+ "4393649844836338524 17495132932599391165 16521868517739667677 5661222119201994079 12972222767581865814 "
+ "11571421678172213790 390646047509318948 11495394588801409501 6525248087626393555 8573656613713326328 "
+ "8142990904973493114 13660896775289126601 17864485770152980871 17445884582967639919 5024202147275854142 "
+ "671342157422491897 8265009237858722295 8608118008319642631 9219145998498351877 4556268366257199356 "
+ "6140599343918232709 1607043993756685523 16576680458922020446 2464344868820317239 11814083511457927881 "
+ "17322648412109813175 1813592025330691328 11477642670415214162 17070997401037634101 2303972879274470462 "
+ "11243326672568121379 3445882696291328363 12839802107382893721 12104481080282265364 2596243342696322386 "
+ "17790946661248112185 16074938231864255959 2956506024687822818 5581797792339843057 16395798806003821452 "
+ "1430185758107085903 18323996060167045894 9286586749315539711 11359324205220105656 13078829860258641519 "
+ "5216000878301612113 9168569275644144847 4001438364296785628 2620979336382597902 17733020247211834335 "
+ "9313982434870058041 14715751745255862275 2010565534563004422 3178715725792479308 1903043159754162676 "
+ "14115336913931799326 2521027868575181436 16014439786100458068 12367605516710589904 209714080394415764 "
+ "1624794657804038156 9935768631398762231 4151749392286673312 7107972358174928160 5660032753352365076 "
+ "18036755596310855736 10390782681895899933 1046990263302077358 7362677844682408358 9462769113228112518 "
+ "16297726431682095630 320526314299713717 3778064933596859787 10885323884409002007 2236973268193149682 "
+ "10878852310456567407 2570883140239125692 2789272264811335056 13059628436710418572 2738898868861677604 "
+ "13893534347610447431 5822583231449316688 6492049503818767408 6244748132635487608 8670290879871155016 "
+ "9029174311949188652 14279734151305645897 15912641926116463991 5242647656726280598 18355120933288718928 "
+ "6020122330537384578 14237586109975592522 4056743947737351776 11111921204484673848 2182608451873446153 "
+ "166740030032355275 6707804438883761942 946217821621517312 17920751901125513000 11238439424546078979 "
+ "9206370960702334859 10367106562255294642 16885700982121403407 5166729472772734774 12763662184890206104 "
+ "5517776110990532992 14067054651980895669 91103470530233291 13189720418381102366 13302941329281487681 "
+ "10033391968256035918 5361508361658563078 9633125940775192275 4070378705292966499 2578880557354159711 "
+ "8213829012818869727 14609024865720281973 5764529456700294955 4441867238869537141 4877335405025028278 "
+ "3719605268964114254 6912476600417191412 12213543493427784533 9772185674504838078 849905513166852628 "
+ "2288907677236569364 13249989292558739703 17027351938809896108 621729935073174643 11783397793000418128 "
+ "6535353235698302382 17474699882318514854 11002055139008150918 18416122529605421577 6901480652243894157 "
+ "10382683051257874235 4029870255315506009 2965992838295436717 6235535328826140656 12567736520515051239 "
+ "1747508188587454664 6183192989050178616 15953675909489391368 10361870655666472156 13992646052760132872 "
+ "18278663149537015721 2341145555999432211 6926699433818498172 13807760559474580665 14452983890116009870 "
+ "11892982856082687091 17809698859762328768 2068107465420962246 7113569128111994893 4401076180719847553 "
+ "11644868941271347491 13124811241626639928 13749911628127226656 17349042196015114201 17667020131618919593 "
+ "5882998107995308535 3994906988813734883 12471577435602039626 6541516905696908495 12651354739990156236 "
+ "2656551093809687944 8349751746116271717 16003262134315245189 16966957802441281611 9846484220369869695 "
+ "289040869684008784 12984844645788078807 15184274406289320733 8585588724744493207 17783251231435661237 "
+ "8433051731677026430 4615759437146530251 7622401495291460413 11623553053949964326 5539054352599742054 "
+ "8346732818944267068 2539102865177431249 12328004780676882137 14682995638339538937 17618899063173726898 "
+ "3560029508967023860 203715062184427611 4331882803273086182 11747937041982829018 14988651560527263167 "
+ "12178986862382858651 3296725696957964696 13476561205033500550 5411341409364572097 6178165960265458773 "
+ "6637587084784255540 2852138145128301044 285104174921779935 3890210696591283091 15859573696692819698 "
+ "4183398296224968094 169433174359854381 11697218950892752694 13577878976046486116 9712904827015064754 "
+ "7617616830643388997 12751433537134599555 12319041734951133946 10239102328951141680 7290886622313676141 "
+ "4739372139955959505 9070795802053402609 10714415902002968511 14446814812660765483 8195026527484180858 "
+ "10331449076861906227 11969763657717422129 11061711610463627412 3682504726309317968 186477037840564317 "
+ "10307292328882458385 11683537113360307954 11300920371008006928 12769285684201107347 14545522978223964713 "
+ "5045546601922369224 202539708050129846 9727300421085682838 10660376569580918468 18521046132102800...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/206423
More information about the libcxx-commits
mailing list