[Mlir-commits] [mlir] Added free-threading CPython mode support in MLIR Python bindings (PR #107103)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Wed Jan 8 02:57:13 PST 2025
github-actions[bot] wrote:
<!--LLVM CODE FORMAT COMMENT: {darker}-->
:warning: Python code formatter, darker found issues in your code. :warning:
<details>
<summary>
You can test this locally with the following command:
</summary>
``````````bash
darker --check --diff -r 32bc029be6265838833623fdd88cc665d5658dc7...18656d9746b003ecf301853286e70e7b437bb828 mlir/test/python/multithreaded_tests.py mlir/test/python/execution_engine.py
``````````
</details>
<details>
<summary>
View the diff from darker here.
</summary>
``````````diff
--- execution_engine.py 2025-01-08 10:52:40.000000 +0000
+++ execution_engine.py 2025-01-08 10:56:42.184259 +0000
@@ -304,11 +304,11 @@
arr = unranked_memref_to_numpy(a, np.float32)
log("Inside callback: ")
log(arr)
with Context():
- # The module takes a subview of the argument memref, casts it to an unranked memref and
+ # The module takes a subview of the argument memref, casts it to an unranked memref and
# calls the callback with it.
module = Module.parse(
r"""
func.func @callback_memref(%arg0: memref<5xf32>) attributes {llvm.emit_c_interface} {
%base_buffer, %offset, %sizes, %strides = memref.extract_strided_metadata %arg0 : memref<5xf32> -> memref<f32>, index, index, index
--- multithreaded_tests.py 2025-01-08 10:52:40.000000 +0000
+++ multithreaded_tests.py 2025-01-08 10:56:42.323244 +0000
@@ -282,11 +282,10 @@
"test_execution_engine__testNanoTime_multi_threaded", # testNanoTime can't run in multiple threads, even with GIL
"test_execution_engine__testSharedLibLoad_multi_threaded", # testSharedLibLoad can't run in multiple threads, even with GIL
"test_dialects_arith_dialect__testArithValue_multi_threaded", # RuntimeError: Value caster is already registered: <class 'dialects/arith_dialect.testArithValue.<locals>.ArithValue'>, even with GIL
"test_ir_dialects__testAppendPrefixSearchPath_multi_threaded", # PyGlobals::setDialectSearchPrefixes is not thread-safe, even with GIL. Strange usage of static PyGlobals vs python exposed _cext.globals
"test_ir_value__testValueCasters_multi_threaded", # RuntimeError: Value caster is already registered: <function testValueCasters.<locals>.dont_cast_int, even with GIL
-
# tests indirectly calling thread-unsafe llvm::raw_ostream
"test_execution_engine__testInvalidModule_multi_threaded", # mlirExecutionEngineCreate calls thread-unsafe llvm::raw_ostream
"test_pass_manager__testPrintIrAfterAll_multi_threaded", # IRPrinterInstrumentation::runAfterPass calls thread-unsafe llvm::raw_ostream
"test_pass_manager__testPrintIrBeforeAndAfterAll_multi_threaded", # IRPrinterInstrumentation::runBeforePass calls thread-unsafe llvm::raw_ostream
"test_pass_manager__testPrintIrLargeLimitElements_multi_threaded", # IRPrinterInstrumentation::runAfterPass calls thread-unsafe llvm::raw_ostream
@@ -295,11 +294,10 @@
"test_dialects_transform_interpreter__include_multi_threaded", # mlir::transform::PrintOp::apply(mlir::transform::TransformRewriter...) calls thread-unsafe llvm::raw_ostream
"test_dialects_transform_interpreter__transform_options_multi_threaded", # mlir::transform::PrintOp::apply(mlir::transform::TransformRewriter...) calls thread-unsafe llvm::raw_ostream
"test_dialects_transform_interpreter__print_self_multi_threaded", # mlir::transform::PrintOp::apply(mlir::transform::TransformRewriter...) call thread-unsafe llvm::raw_ostream
"test_ir_diagnostic_handler__testDiagnosticCallbackException_multi_threaded", # mlirEmitError calls thread-unsafe llvm::raw_ostream
"test_ir_module__testParseSuccess_multi_threaded", # mlirOperationDump calls thread-unsafe llvm::raw_ostream
-
# False-positive TSAN detected race in llvm::RuntimeDyldELF::registerEHFrames()
# Details: https://github.com/llvm/llvm-project/pull/107103/files#r1905726947
"test_execution_engine__testCapsule_multi_threaded",
"test_execution_engine__testDumpToObjectFile_multi_threaded",
]
@@ -321,29 +319,26 @@
"test_execution_engine__testRankedMemRefWithOffsetCallback_multi_threaded",
"test_execution_engine__testUnrankedMemRefCallback_multi_threaded",
"test_execution_engine__testUnrankedMemRefWithOffsetCallback_multi_threaded",
# dialects tests
"test_dialects_memref__testSubViewOpInferReturnTypeExtensiveSlicing_multi_threaded", # Related to ctypes data races
-
"test_dialects_transform_interpreter__print_other_multi_threaded", # Fatal Python error: Aborted or mlir::transform::PrintOp::apply(mlir::transform::TransformRewriter...) is not thread-safe
-
"test_dialects_gpu_module-to-binary-rocdl__testGPUToASMBin_multi_threaded", # Due to global llvm-project/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp::GCNTrackers variable mutation
"test_dialects_gpu_module-to-binary-nvvm__testGPUToASMBin_multi_threaded",
"test_dialects_gpu_module-to-binary-nvvm__testGPUToLLVMBin_multi_threaded",
"test_dialects_gpu_module-to-binary-rocdl__testGPUToLLVMBin_multi_threaded",
-
# integration tests
"test_integration_dialects_linalg_opsrun__test_elemwise_builtin_multi_threaded", # Related to ctypes data races
"test_integration_dialects_linalg_opsrun__test_elemwise_generic_multi_threaded", # Related to ctypes data races
"test_integration_dialects_linalg_opsrun__test_fill_builtin_multi_threaded", # ctypes
"test_integration_dialects_linalg_opsrun__test_fill_generic_multi_threaded", # ctypes
- "test_integration_dialects_linalg_opsrun__test_fill_rng_builtin_multi_threaded", # ctypes
- "test_integration_dialects_linalg_opsrun__test_fill_rng_generic_multi_threaded", # ctypes
- "test_integration_dialects_linalg_opsrun__test_max_pooling_builtin_multi_threaded", # ctypes
- "test_integration_dialects_linalg_opsrun__test_max_pooling_generic_multi_threaded", # ctypes
- "test_integration_dialects_linalg_opsrun__test_min_pooling_builtin_multi_threaded", # ctypes
- "test_integration_dialects_linalg_opsrun__test_min_pooling_generic_multi_threaded", # ctypes
+ "test_integration_dialects_linalg_opsrun__test_fill_rng_builtin_multi_threaded", # ctypes
+ "test_integration_dialects_linalg_opsrun__test_fill_rng_generic_multi_threaded", # ctypes
+ "test_integration_dialects_linalg_opsrun__test_max_pooling_builtin_multi_threaded", # ctypes
+ "test_integration_dialects_linalg_opsrun__test_max_pooling_generic_multi_threaded", # ctypes
+ "test_integration_dialects_linalg_opsrun__test_min_pooling_builtin_multi_threaded", # ctypes
+ "test_integration_dialects_linalg_opsrun__test_min_pooling_generic_multi_threaded", # ctypes
]
def add_existing_tests(test_modules, test_prefix: str = "_original_test"):
def decorator(test_cls):
@@ -395,11 +390,11 @@
def get_output() -> str:
if captured is None:
raise ValueError("get_output() called while the context is active.")
return captured
- with tempfile.NamedTemporaryFile(mode="w+", encoding='utf-8') as f:
+ with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as f:
original_fd = os.dup(fp.fileno())
os.dup2(f.fileno(), fp.fileno())
try:
yield get_output
finally:
@@ -408,10 +403,11 @@
os.fsync(fp.fileno())
f.seek(0)
captured = f.read()
os.dup2(original_fd, fp.fileno())
+
capture_stdout = partial(_capture_output, sys.stdout)
capture_stderr = partial(_capture_output, sys.stderr)
def multi_threaded(
@@ -435,13 +431,11 @@
test_name.replace(multithreaded_test_postfix, "") in name
for test_name in skip_tests
):
continue
- def multi_threaded_test_fn(
- self, *args, __test_fn__=test_fn, **kwargs
- ):
+ def multi_threaded_test_fn(self, *args, __test_fn__=test_fn, **kwargs):
with capture_stdout(), capture_stderr() as get_output:
barrier = threading.Barrier(num_workers)
def closure():
barrier.wait()
@@ -467,11 +461,13 @@
f"ThreadSanitizer reported warnings:\n{captured}"
)
test_new_name = f"{name}{multithreaded_test_postfix}"
if xfail_tests is not None and test_new_name in xfail_tests:
- multi_threaded_test_fn = unittest.expectedFailure(multi_threaded_test_fn)
+ multi_threaded_test_fn = unittest.expectedFailure(
+ multi_threaded_test_fn
+ )
setattr(test_cls, test_new_name, multi_threaded_test_fn)
return test_cls
@@ -484,11 +480,10 @@
skip_tests=TESTS_TO_SKIP,
xfail_tests=TESTS_TO_XFAIL,
)
@add_existing_tests(test_modules=TEST_MODULES, test_prefix="_original_test")
class TestAllMultiThreaded(unittest.TestCase):
-
@classmethod
def tearDownClass(cls):
if hasattr(cls, "output_folder"):
cls.output_folder.cleanup()
``````````
</details>
https://github.com/llvm/llvm-project/pull/107103
More information about the Mlir-commits
mailing list