From f7421f8c9cc620f829b3e3ce86d5ff12a86c1412 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Wed, 24 Jul 2024 17:19:30 +0100 Subject: [PATCH 01/37] slightly more thread safe gc --- src/C/pointers.jl | 2 ++ src/GC/GC.jl | 63 +++++++++++++++++++++++++----------- test/finalize_test_script.jl | 24 ++++++++++++++ 3 files changed, 71 insertions(+), 18 deletions(-) create mode 100644 test/finalize_test_script.jl diff --git a/src/C/pointers.jl b/src/C/pointers.jl index dd0476fc..6faabb60 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -22,6 +22,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( :PyEval_RestoreThread => (Ptr{Cvoid},) => Cvoid, :PyGILState_Ensure => () => PyGILState_STATE, :PyGILState_Release => (PyGILState_STATE,) => Cvoid, + :PyGILState_GetThisThreadState => () => Ptr{Cvoid}, + :PyGILState_Check => () => Cint, # IMPORT :PyImport_ImportModule => (Ptr{Cchar},) => PyPtr, :PyImport_Import => (PyPtr,) => PyPtr, diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 0d1fa9a8..48e70544 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -39,25 +39,36 @@ Like most PythonCall functions, you must only call this from the main thread. """ function enable() ENABLED[] = true - if !isempty(QUEUE) - C.with_gil(false) do - for ptr in QUEUE - if ptr != C.PyNULL - C.Py_DecRef(ptr) - end - end + if !isempty(QUEUE) && C.PyGILState_Check() == 1 + free_queue() + end + return +end + +function free_queue() + for ptr in QUEUE + if ptr != C.PyNULL + C.Py_DecRef(ptr) end end empty!(QUEUE) - return + nothing +end + +function gc() + if ENABLED[] && C.PyGILState_Check() == 1 + free_queue() + true + else + false + end end function enqueue(ptr::C.PyPtr) if ptr != C.PyNULL && C.CTX.is_initialized - if ENABLED[] - C.with_gil(false) do - C.Py_DecRef(ptr) - end + if ENABLED[] && C.PyGILState_Check() == 1 + C.Py_DecRef(ptr) + isempty(QUEUE) || free_queue() else push!(QUEUE, ptr) end @@ -67,14 +78,13 @@ end function enqueue_all(ptrs) if C.CTX.is_initialized - if ENABLED[] - C.with_gil(false) do - for ptr in ptrs - if ptr != C.PyNULL - C.Py_DecRef(ptr) - end + if ENABLED[] && C.PyGILState_Check() == 1 + for ptr in ptrs + if ptr != C.PyNULL + C.Py_DecRef(ptr) end end + isempty(QUEUE) || free_queue() else append!(QUEUE, ptrs) end @@ -82,4 +92,21 @@ function enqueue_all(ptrs) return end +mutable struct GCHook + function GCHook() + finalizer(_gchook_finalizer, new()) + end +end + +function _gchook_finalizer(x) + gc() + finalizer(_gchook_finalizer, x) + nothing +end + +function __init__() + GCHook() + nothing +end + end # module GC diff --git a/test/finalize_test_script.jl b/test/finalize_test_script.jl new file mode 100644 index 00000000..28d4dd72 --- /dev/null +++ b/test/finalize_test_script.jl @@ -0,0 +1,24 @@ +using PythonCall + +# This would consistently segfault pre-GC-thread-safety +let + pyobjs = map(pylist, 1:100) + Threads.@threads for obj in pyobjs + finalize(obj) + end +end + +@show PythonCall.GC.ENABLED[] +@show length(PythonCall.GC.QUEUE) +GC.gc(false) +# with GCHook, the queue should be empty now (the above gc() triggered GCHook to clear the PythonCall QUEUE) +# without GCHook, gc() has no effect on the QUEUE +@show length(PythonCall.GC.QUEUE) +GC.gc(false) +@show length(PythonCall.GC.QUEUE) +GC.gc(false) +@show length(PythonCall.GC.QUEUE) +# with GCHook this is not necessary, GC.gc() is enough +# without GCHook, this is required to free any objects in the PythonCall QUEUE +PythonCall.GC.gc() +@show length(PythonCall.GC.QUEUE) From 3bcd028cd81c5c547f82dcc7cfa49cf7d71c06a9 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Mon, 29 Jul 2024 21:49:01 +0100 Subject: [PATCH 02/37] use Channel not Vector and make disable/enable a no-op --- src/GC/GC.jl | 100 +++++++++++++++++++---------------- test/finalize_test_script.jl | 17 +++--- 2 files changed, 62 insertions(+), 55 deletions(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 48e70544..357e98d0 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -3,93 +3,97 @@ Garbage collection of Python objects. -See `disable` and `enable`. +See [`enable`](@ref), [`disable`](@ref) and [`gc`](@ref). """ module GC using ..C: C -const ENABLED = Ref(true) -const QUEUE = C.PyPtr[] +const QUEUE = Channel{C.PyPtr}(Inf) +const HOOK = WeakRef() """ PythonCall.GC.disable() -Disable the PythonCall garbage collector. +Do nothing. -This means that whenever a Python object owned by Julia is finalized, it is not immediately -freed but is instead added to a queue of objects to free later when `enable()` is called. +!!! note -Like most PythonCall functions, you must only call this from the main thread. + Historically this would disable the PythonCall garbage collector. This was required + for safety in multi-threaded code but is no longer needed, so this is now a no-op. """ -function disable() - ENABLED[] = false - return -end +disable() = nothing """ PythonCall.GC.enable() -Re-enable the PythonCall garbage collector. +Do nothing. -This frees any Python objects which were finalized while the GC was disabled, and allows -objects finalized in the future to be freed immediately. +!!! note -Like most PythonCall functions, you must only call this from the main thread. + Historically this would enable the PythonCall garbage collector. This was required + for safety in multi-threaded code but is no longer needed, so this is now a no-op. """ -function enable() - ENABLED[] = true - if !isempty(QUEUE) && C.PyGILState_Check() == 1 - free_queue() - end - return -end +enable() = nothing -function free_queue() - for ptr in QUEUE - if ptr != C.PyNULL - C.Py_DecRef(ptr) - end +""" + PythonCall.GC.gc() + +Free any Python objects waiting to be freed. + +These are objects that were finalized from a thread that was not holding the Python +GIL at the time. + +Like most PythonCall functions, this must only be called from the main thread (i.e. the +thread currently holding the Python GIL.) +""" +function gc() + if C.CTX.is_initialized + unsafe_free_queue() end - empty!(QUEUE) nothing end -function gc() - if ENABLED[] && C.PyGILState_Check() == 1 - free_queue() - true - else - false +function unsafe_free_queue() + if isready(QUEUE) + @lock QUEUE while isready(QUEUE) + ptr = take!(QUEUE) + if ptr != C.PyNULL + C.Py_DecRef(ptr) + end + end end + nothing end function enqueue(ptr::C.PyPtr) if ptr != C.PyNULL && C.CTX.is_initialized - if ENABLED[] && C.PyGILState_Check() == 1 + if C.PyGILState_Check() == 1 C.Py_DecRef(ptr) - isempty(QUEUE) || free_queue() + unsafe_free_queue() else - push!(QUEUE, ptr) + put!(QUEUE, ptr) end end - return + nothing end function enqueue_all(ptrs) - if C.CTX.is_initialized - if ENABLED[] && C.PyGILState_Check() == 1 + if any(ptr -> ptr != C.PYNULL, ptrs) && C.CTX.is_initialized + if C.PyGILState_Check() == 1 for ptr in ptrs if ptr != C.PyNULL C.Py_DecRef(ptr) end end - isempty(QUEUE) || free_queue() + unsafe_free_queue() else - append!(QUEUE, ptrs) + for ptr in ptrs + put!(QUEUE, ptr) + end end end - return + nothing end mutable struct GCHook @@ -99,13 +103,17 @@ mutable struct GCHook end function _gchook_finalizer(x) - gc() - finalizer(_gchook_finalizer, x) + if C.CTX.is_initialized + finalizer(_gchook_finalizer, x) + if isready(QUEUE) && C.PyGILState_Check() == 1 + unsafe_free_queue() + end + end nothing end function __init__() - GCHook() + HOOK.value = GCHook() nothing end diff --git a/test/finalize_test_script.jl b/test/finalize_test_script.jl index 28d4dd72..41efbc79 100644 --- a/test/finalize_test_script.jl +++ b/test/finalize_test_script.jl @@ -8,17 +8,16 @@ let end end -@show PythonCall.GC.ENABLED[] -@show length(PythonCall.GC.QUEUE) -GC.gc(false) +@show isready(PythonCall.GC.QUEUE) +GC.gc() # with GCHook, the queue should be empty now (the above gc() triggered GCHook to clear the PythonCall QUEUE) # without GCHook, gc() has no effect on the QUEUE -@show length(PythonCall.GC.QUEUE) -GC.gc(false) -@show length(PythonCall.GC.QUEUE) -GC.gc(false) -@show length(PythonCall.GC.QUEUE) +@show isready(PythonCall.GC.QUEUE) +GC.gc() +@show isready(PythonCall.GC.QUEUE) +GC.gc() +@show isready(PythonCall.GC.QUEUE) # with GCHook this is not necessary, GC.gc() is enough # without GCHook, this is required to free any objects in the PythonCall QUEUE PythonCall.GC.gc() -@show length(PythonCall.GC.QUEUE) +@show isready(PythonCall.GC.QUEUE) From 8ca05c9ac7bc2beaed7cc86a00a2a7719d201cd2 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Mon, 29 Jul 2024 21:56:30 +0100 Subject: [PATCH 03/37] document GCHook --- src/GC/GC.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 357e98d0..0163cb0f 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -96,6 +96,16 @@ function enqueue_all(ptrs) nothing end +""" + GCHook() + +An immortal object which frees any pending Python objects when Julia's GC runs. + +This works by creating it but not holding any strong reference to it, so it is eligible +to be finalized by Julia's GC. The finalizer empties the PythonCall GC queue if +possible. The finalizer also re-attaches itself, so the object does not actually get +collected and so the finalizer will run again at next GC. +""" mutable struct GCHook function GCHook() finalizer(_gchook_finalizer, new()) From e230ce9a3b8cb5f01e730fd22bb3138c8f5f234b Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Mon, 29 Jul 2024 22:10:27 +0100 Subject: [PATCH 04/37] cannot lock channels on julia 1.6 --- src/GC/GC.jl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 0163cb0f..771b4050 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -55,12 +55,10 @@ function gc() end function unsafe_free_queue() - if isready(QUEUE) - @lock QUEUE while isready(QUEUE) - ptr = take!(QUEUE) - if ptr != C.PyNULL - C.Py_DecRef(ptr) - end + while isready(QUEUE) + ptr = take!(QUEUE) + if ptr != C.PyNULL + C.Py_DecRef(ptr) end end nothing From a36d7c09314d101cdd60cecaade03cb6840a0b80 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Wed, 31 Jul 2024 19:25:41 +0100 Subject: [PATCH 05/37] revert to using a vector for the queue --- src/GC/GC.jl | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 771b4050..a076f3ed 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -9,7 +9,8 @@ module GC using ..C: C -const QUEUE = Channel{C.PyPtr}(Inf) +const QUEUE = C.PyPtr[] +const QUEUE_LOCK = Threads.SpinLock() const HOOK = WeakRef() """ @@ -55,12 +56,14 @@ function gc() end function unsafe_free_queue() - while isready(QUEUE) - ptr = take!(QUEUE) + lock(QUEUE_LOCK) + for ptr in QUEUE if ptr != C.PyNULL C.Py_DecRef(ptr) end end + empty!(QUEUE) + unlock(QUEUE_LOCK) nothing end @@ -68,9 +71,13 @@ function enqueue(ptr::C.PyPtr) if ptr != C.PyNULL && C.CTX.is_initialized if C.PyGILState_Check() == 1 C.Py_DecRef(ptr) - unsafe_free_queue() + if !isempty(QUEUE) + unsafe_free_queue() + end else - put!(QUEUE, ptr) + lock(QUEUE_LOCK) + push!(QUEUE, ptr) + unlock(QUEUE_LOCK) end end nothing @@ -84,11 +91,13 @@ function enqueue_all(ptrs) C.Py_DecRef(ptr) end end - unsafe_free_queue() - else - for ptr in ptrs - put!(QUEUE, ptr) + if !isempty(QUEUE) + unsafe_free_queue() end + else + lock(QUEUE_LOCK) + append!(QUEUE, ptrs) + unlock(QUEUE_LOCK) end end nothing @@ -113,7 +122,7 @@ end function _gchook_finalizer(x) if C.CTX.is_initialized finalizer(_gchook_finalizer, x) - if isready(QUEUE) && C.PyGILState_Check() == 1 + if !isempty(QUEUE) && C.PyGILState_Check() == 1 unsafe_free_queue() end end From a5a2c96bfa41bbbe814f466f848f754ea67bfc5d Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Wed, 31 Jul 2024 19:44:34 +0100 Subject: [PATCH 06/37] restore test script --- test/finalize_test_script.jl | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/finalize_test_script.jl b/test/finalize_test_script.jl index 41efbc79..ecacad9e 100644 --- a/test/finalize_test_script.jl +++ b/test/finalize_test_script.jl @@ -7,17 +7,3 @@ let finalize(obj) end end - -@show isready(PythonCall.GC.QUEUE) -GC.gc() -# with GCHook, the queue should be empty now (the above gc() triggered GCHook to clear the PythonCall QUEUE) -# without GCHook, gc() has no effect on the QUEUE -@show isready(PythonCall.GC.QUEUE) -GC.gc() -@show isready(PythonCall.GC.QUEUE) -GC.gc() -@show isready(PythonCall.GC.QUEUE) -# with GCHook this is not necessary, GC.gc() is enough -# without GCHook, this is required to free any objects in the PythonCall QUEUE -PythonCall.GC.gc() -@show isready(PythonCall.GC.QUEUE) From f021072db70668a60364e51b6a806cdf3f46658d Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 1 Aug 2024 21:32:36 +0100 Subject: [PATCH 07/37] combine queue into a single item --- src/GC/GC.jl | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index a076f3ed..898a7eac 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -9,8 +9,7 @@ module GC using ..C: C -const QUEUE = C.PyPtr[] -const QUEUE_LOCK = Threads.SpinLock() +const QUEUE = (; items = C.PyPtr[], lock = Threads.SpinLock()) const HOOK = WeakRef() """ @@ -56,14 +55,14 @@ function gc() end function unsafe_free_queue() - lock(QUEUE_LOCK) - for ptr in QUEUE + lock(QUEUE.lock) + for ptr in QUEUE.items if ptr != C.PyNULL C.Py_DecRef(ptr) end end - empty!(QUEUE) - unlock(QUEUE_LOCK) + empty!(QUEUE.items) + unlock(QUEUE.lock) nothing end @@ -71,13 +70,13 @@ function enqueue(ptr::C.PyPtr) if ptr != C.PyNULL && C.CTX.is_initialized if C.PyGILState_Check() == 1 C.Py_DecRef(ptr) - if !isempty(QUEUE) + if !isempty(QUEUE.items) unsafe_free_queue() end else - lock(QUEUE_LOCK) - push!(QUEUE, ptr) - unlock(QUEUE_LOCK) + lock(QUEUE.lock) + push!(QUEUE.items, ptr) + unlock(QUEUE.lock) end end nothing @@ -91,13 +90,13 @@ function enqueue_all(ptrs) C.Py_DecRef(ptr) end end - if !isempty(QUEUE) + if !isempty(QUEUE.items) unsafe_free_queue() end else - lock(QUEUE_LOCK) - append!(QUEUE, ptrs) - unlock(QUEUE_LOCK) + lock(QUEUE.lock) + append!(QUEUE.items, ptrs) + unlock(QUEUE.lock) end end nothing @@ -122,7 +121,7 @@ end function _gchook_finalizer(x) if C.CTX.is_initialized finalizer(_gchook_finalizer, x) - if !isempty(QUEUE) && C.PyGILState_Check() == 1 + if !isempty(QUEUE.items) && C.PyGILState_Check() == 1 unsafe_free_queue() end end From 4b3bd65bf89e5edc0746c6e67cfd98528f19c003 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 1 Aug 2024 21:37:12 +0100 Subject: [PATCH 08/37] prefer Fix2 over anonymous function --- src/GC/GC.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 898a7eac..9ac93bcf 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -83,7 +83,7 @@ function enqueue(ptr::C.PyPtr) end function enqueue_all(ptrs) - if any(ptr -> ptr != C.PYNULL, ptrs) && C.CTX.is_initialized + if any(!=(C.PYNULL), ptrs) && C.CTX.is_initialized if C.PyGILState_Check() == 1 for ptr in ptrs if ptr != C.PyNULL From 56aa9bc863498337f55ff720cbeadfec984009ca Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 1 Aug 2024 21:53:02 +0100 Subject: [PATCH 09/37] update docs --- docs/src/faq.md | 30 +++++++++++++++++------------- docs/src/releasenotes.md | 8 ++++++++ src/GC/GC.jl | 2 +- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/docs/src/faq.md b/docs/src/faq.md index 981aa1ed..b51717d0 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -4,19 +4,23 @@ No. -Some rules if you are writing multithreaded code: -- Only call Python functions from the first thread. -- You probably also need to call `PythonCall.GC.disable()` on the main thread before any - threaded block of code. Remember to call `PythonCall.GC.enable()` again afterwards. - (This is because Julia finalizers can be called from any thread.) -- Julia intentionally causes segmentation faults as part of the GC safepoint mechanism. - If unhandled, these segfaults will result in termination of the process. To enable signal handling, - set `PYTHON_JULIACALL_HANDLE_SIGNALS=yes` before any calls to import juliacall. This is equivalent - to starting julia with `julia --handle-signals=yes`, the default behavior in Julia. - See discussion [here](https://github.com/JuliaPy/PythonCall.jl/issues/219#issuecomment-1605087024) for more information. -- You may still encounter problems. - -Related issues: [#201](https://github.com/JuliaPy/PythonCall.jl/issues/201), [#202](https://github.com/JuliaPy/PythonCall.jl/issues/202) +However it is safe to use PythonCall with Julia with multiple threads, provided you only +call Python code from the first thread. (Before v0.9.22, tricks such as disabling the +garbage collector were required.) + +From Python, to use JuliaCall with multiple threads you probably need to set +[`PYTHON_JULIACALL_HANDLE_SIGNALS=yes`](@ref julia-config) before importing JuliaCall. +This is because Julia intentionally causes segmentation faults as part of the GC +safepoint mechanism. If unhandled, these segfaults will result in termination of the +process. This is equivalent to starting julia with `julia --handle-signals=yes`, the +default behavior in Julia. See discussion +[here](https://github.com/JuliaPy/PythonCall.jl/issues/219#issuecomment-1605087024) +for more information. + +Related issues: +[#201](https://github.com/JuliaPy/PythonCall.jl/issues/201), +[#202](https://github.com/JuliaPy/PythonCall.jl/issues/202), +[#529](https://github.com/JuliaPy/PythonCall.jl/pull/529) ## Issues when Numpy arrays are expected diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 500dbf98..33141da8 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,5 +1,13 @@ # Release Notes +## Unreleased +* Finalizers are now thread-safe, meaning PythonCall now works in the presence of + multi-threaded Julia code. Previously, tricks such as disabling the garbage collector + were required. Python code must still be called on the main thread. +* `GC.disable()` and `GC.enable()` are now a no-op and deprecated since they are no + longer required for thread-safety. These will be removed in v1. +* Adds `GC.gc()`. + ## 0.9.21 (2024-07-20) * `Serialization.serialize` can use `dill` instead of `pickle` by setting the env var `JULIA_PYTHONCALL_PICKLE=dill`. * `numpy.bool_` can now be converted to `Bool` and other number types. diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 9ac93bcf..2a8d53ac 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -3,7 +3,7 @@ Garbage collection of Python objects. -See [`enable`](@ref), [`disable`](@ref) and [`gc`](@ref). +See [`gc`](@ref). """ module GC From 4fdcf310d75c92edd757233e2f5f267a09f6a59a Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 1 Aug 2024 22:02:40 +0100 Subject: [PATCH 10/37] test multithreaded --- .github/workflows/tests-nightly.yml | 1 + .github/workflows/tests.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/tests-nightly.yml b/.github/workflows/tests-nightly.yml index 6a443463..f73f9a7d 100644 --- a/.github/workflows/tests-nightly.yml +++ b/.github/workflows/tests-nightly.yml @@ -38,6 +38,7 @@ jobs: - uses: julia-actions/julia-runtest@v1 env: JULIA_DEBUG: PythonCall + JULIA_NUM_THREADS: '2' - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a3462b48..bc4d52d0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,6 +43,7 @@ jobs: uses: julia-actions/julia-runtest@v1 env: JULIA_DEBUG: PythonCall + JULIA_NUM_THREADS: '2' - name: Process coverage uses: julia-actions/julia-processcoverage@v1 - name: Upload coverage to Codecov @@ -82,6 +83,8 @@ jobs: - name: Run tests run: | pytest -s --nbval --cov=pysrc ./pytest/ + env: + PYTHON_JULIACALL_THREADS: '2' - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 env: From 9051769376a142d0811053419d9bbbf2d428d6d9 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 1 Aug 2024 22:13:17 +0100 Subject: [PATCH 11/37] test gc from python --- pytest/test_all.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pytest/test_all.py b/pytest/test_all.py index c6cff009..f94f91f6 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -75,3 +75,28 @@ def test_issue_433(): """ ) assert out == 25 + +def test_julia_gc(): + from juliacall import Main as jl + # We make a bunch of python objects with no reference to them, + # then call GC to try to finalize them. + # We want to make sure we don't segfault. + # Here we can (manually) verify that the background task is running successfully, + # by seeing the printout "Python GC (100 items): 0.000000 seconds." + # We also programmatically check things are working by verifying the queue is empty. + # Debugging note: if you get segfaults, then run the tests with + # `PYTHON_JULIACALL_HANDLE_SIGNALS=yes python3 -X faulthandler -m pytest -p no:faulthandler -s --nbval --cov=pysrc ./pytest/` + # in order to recover a bit more information from the segfault. + jl.seval( + """ + using PythonCall, Test + let + pyobjs = map(pylist, 1:100) + Threads.@threads for obj in pyobjs + finalize(obj) + end + end + GC.gc() + @test isempty(PythonCall.GC.QUEUE.items) + """ + ) From 13cc34648e8098b227a2d1d3778aa134196ad594 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 1 Aug 2024 22:22:36 +0100 Subject: [PATCH 12/37] add gc tests --- pytest/test_all.py | 1 + test/GC.jl | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/pytest/test_all.py b/pytest/test_all.py index f94f91f6..0916e918 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -96,6 +96,7 @@ def test_julia_gc(): finalize(obj) end end + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() @test isempty(PythonCall.GC.QUEUE.items) """ diff --git a/test/GC.jl b/test/GC.jl index 46409041..93454100 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1 +1,32 @@ -# TODO +@testset "201: GC segfaults" begin + # https://github.com/JuliaPy/PythonCall.jl/issues/201 + # This should not segfault! + cmd = Base.julia_cmd() + path = joinpath(@__DIR__, "finalize_test_script.jl") + p = run(`$cmd -t2 --project $path`) + @test p.exitcode == 0 +end + +@testset "GC.gc()" begin + let + pyobjs = map(pylist, 1:100) + Threads.@threads for obj in pyobjs + finalize(obj) + end + end + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() + @test isempty(PythonCall.GC.QUEUE.items) +end + +@testset "GC.GCHook" begin + let + pyobjs = map(pylist, 1:100) + Threads.@threads for obj in pyobjs + finalize(obj) + end + end + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + GC.gc() + @test isempty(PythonCall.GC.QUEUE.items) +end From 45bc71f40df841d60c00066847e7c672a8ab3b1c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 1 Aug 2024 22:36:59 +0100 Subject: [PATCH 13/37] fix test --- pytest/test_all.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest/test_all.py b/pytest/test_all.py index 0916e918..f94f91f6 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -96,7 +96,6 @@ def test_julia_gc(): finalize(obj) end end - Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() @test isempty(PythonCall.GC.QUEUE.items) """ From 4ec7def7a6ae2d54482781e5525646e8196b050c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 17:53:34 +0100 Subject: [PATCH 14/37] add deprecation warnings --- src/GC/GC.jl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 2a8d53ac..365ab0fe 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -22,7 +22,13 @@ Do nothing. Historically this would disable the PythonCall garbage collector. This was required for safety in multi-threaded code but is no longer needed, so this is now a no-op. """ -disable() = nothing +function disable() + Base.depwarn( + "disabling the PythonCall GC is no longer needed for thread-safety", + :disable, + ) + nothing +end """ PythonCall.GC.enable() @@ -34,7 +40,13 @@ Do nothing. Historically this would enable the PythonCall garbage collector. This was required for safety in multi-threaded code but is no longer needed, so this is now a no-op. """ -enable() = nothing +function enable() + Base.depwarn( + "disabling the PythonCall GC is no longer needed for thread-safety", + :enable, + ) + nothing +end """ PythonCall.GC.gc() From eb6b9f0dc584ef2d34b71ac348e533ac52f6ae3e Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 17:54:49 +0100 Subject: [PATCH 15/37] safer locking (plus explanatory comments) --- src/GC/GC.jl | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 365ab0fe..98908b5d 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -67,28 +67,36 @@ function gc() end function unsafe_free_queue() - lock(QUEUE.lock) - for ptr in QUEUE.items - if ptr != C.PyNULL - C.Py_DecRef(ptr) + Base.@lock QUEUE.lock begin + for ptr in QUEUE.items + if ptr != C.PyNULL + C.Py_DecRef(ptr) + end end + empty!(QUEUE.items) end - empty!(QUEUE.items) - unlock(QUEUE.lock) nothing end function enqueue(ptr::C.PyPtr) + # If the ptr is NULL there is nothing to free. + # If C.CTX.is_initialized is false then the Python interpreter hasn't started yet + # or has been finalized; either way attempting to free will cause an error. if ptr != C.PyNULL && C.CTX.is_initialized if C.PyGILState_Check() == 1 + # If the current thread holds the GIL, then we can immediately free. C.Py_DecRef(ptr) + # We may as well also free any other enqueued objects. if !isempty(QUEUE.items) unsafe_free_queue() end else - lock(QUEUE.lock) - push!(QUEUE.items, ptr) - unlock(QUEUE.lock) + # Otherwise we push the pointer onto the queue to be freed later, either: + # (a) If a future Python object is finalized on the thread holding the GIL + # in the branch above. + # (b) If the GCHook() object below is finalized in an ordinary GC. + # (c) If the user calls PythonCall.GC.gc(). + Base.@lock QUEUE.lock push!(QUEUE.items, ptr) end end nothing @@ -106,9 +114,7 @@ function enqueue_all(ptrs) unsafe_free_queue() end else - lock(QUEUE.lock) - append!(QUEUE.items, ptrs) - unlock(QUEUE.lock) + Base.@lock QUEUE.lock append!(QUEUE.items, ptrs) end end nothing From a68015ecc145aa32eea71ccc8149258e68fe9303 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 17:55:06 +0100 Subject: [PATCH 16/37] ref of weakref --- src/GC/GC.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 98908b5d..e7e992a6 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -10,7 +10,7 @@ module GC using ..C: C const QUEUE = (; items = C.PyPtr[], lock = Threads.SpinLock()) -const HOOK = WeakRef() +const HOOK = Ref{WeakRef}() """ PythonCall.GC.disable() @@ -147,7 +147,7 @@ function _gchook_finalizer(x) end function __init__() - HOOK.value = GCHook() + HOOK[] = WeakRef(GCHook()) nothing end From ab560ac94c9026d120d8398a4f05eea22470a455 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 18:03:00 +0100 Subject: [PATCH 17/37] SpinLock -> ReentrantLock --- src/GC/GC.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index e7e992a6..3b25b760 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -9,7 +9,7 @@ module GC using ..C: C -const QUEUE = (; items = C.PyPtr[], lock = Threads.SpinLock()) +const QUEUE = (; items = C.PyPtr[], lock = ReentrantLock()) const HOOK = Ref{WeakRef}() """ From cd4db5c2f47fdf300916f68836de888784cf067c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 18:34:06 +0100 Subject: [PATCH 18/37] SpinLock -> ReentrantLock --- src/GC/GC.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 3b25b760..e7e992a6 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -9,7 +9,7 @@ module GC using ..C: C -const QUEUE = (; items = C.PyPtr[], lock = ReentrantLock()) +const QUEUE = (; items = C.PyPtr[], lock = Threads.SpinLock()) const HOOK = Ref{WeakRef}() """ From 980701e4b21da9ef7409e9bef78b0b351bf2d189 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 20:14:46 +0100 Subject: [PATCH 19/37] add PythonCall.GIL --- src/GIL/GIL.jl | 96 +++++++++++++++++++++++++++++++++++++++++++++++ src/PythonCall.jl | 1 + 2 files changed, 97 insertions(+) create mode 100644 src/GIL/GIL.jl diff --git a/src/GIL/GIL.jl b/src/GIL/GIL.jl new file mode 100644 index 00000000..c1e89f23 --- /dev/null +++ b/src/GIL/GIL.jl @@ -0,0 +1,96 @@ +""" + module PythonCall.GIL + +Handling the Python Global Interpreter Lock. + +See [`lock`](@ref), [`@lock`](@ref), [`release`](@ref) and [`@release`](@ref). +""" +module GIL + +using ..C: C + +""" + lock(f) + +Acquire the GIL, compute `f()`, release the GIL, then return the result of `f()`. + +Use this to run Python code from threads that do not currently hold the GIL, such as new +threads. Since the main Julia thread holds the GIL by default, you will need to +[`release`](@ref) the GIL before using this function. + +See [`@lock`](@ref) for the macro form. +""" +function lock(f) + state = C.PyGILState_Ensure() + try + f() + finally + C.PyGILState_Release(state) + end +end + +""" + @lock expr + +Acquire the GIL, compute `expr`, release the GIL, then return the result of `expr`. + +Use this to run Python code from threads that do not currently hold the GIL, such as new +threads. Since the main Julia thread holds the GIL by default, you will need to +[`@release`](@ref) the GIL before using this function. + +The macro equivalent of [`lock`](@ref). +""" +macro lock(expr) + quote + state = C.PyGILState_Ensure() + try + $(esc(expr)) + finally + C.PyGILState_Release(state) + end + end +end + +""" + release(f) + +Release the GIL, compute `f()`, re-acquire the GIL, then return the result of `f()`. + +Use this to run non-Python code with the GIL released, so allowing another thread to run +Python code. That other thread can be a Julia thread, which must acquire the GIL using +[`lock`](@ref). + +See [`@release`](@ref) for the macro form. +""" +function release(f) + state = C.PyEval_SaveThread() + try + f() + finally + C.PyEval_RestoreThread(state) + end +end + +""" + @release expr + +Release the GIL, compute `expr`, re-acquire the GIL, then return the result of `expr`. + +Use this to run non-Python code with the GIL released, so allowing another thread to run +Python code. That other thread can be a Julia thread, which must acquire the GIL using +[`@lock`](@ref). + +The macro equivalent of [`release`](@ref). +""" +macro release(expr) + quote + state = C.PyEval_SaveThread() + try + $(esc(expr)) + finally + C.PyEval_RestoreThread(state) + end + end +end + +end diff --git a/src/PythonCall.jl b/src/PythonCall.jl index 68695456..a2e7c3d8 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -5,6 +5,7 @@ const ROOT_DIR = dirname(@__DIR__) include("Utils/Utils.jl") include("C/C.jl") +include("GIL/GIL.jl") include("GC/GC.jl") include("Core/Core.jl") include("Convert/Convert.jl") From f2bc4ce429a0b97070c92e384dab3dfce39cf510 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 20:14:56 +0100 Subject: [PATCH 20/37] add tests for PythonCall.GIL --- test/GIL.jl | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 test/GIL.jl diff --git a/test/GIL.jl b/test/GIL.jl new file mode 100644 index 00000000..a0249282 --- /dev/null +++ b/test/GIL.jl @@ -0,0 +1,39 @@ +@testitem "release and lock" begin + # This calls Python's time.sleep(1) twice concurrently. Since sleep() releases the + # GIL, these can happen in parallel if Julia has at least 2 threads. + function threaded_sleep() + PythonCall.GIL.release() do + Threads.@threads for i = 1:2 + PythonCall.GIL.lock() do + pyimport("time").sleep(1) + end + end + end + end + # one run to ensure it's compiled + threaded_sleep() + # now time it + t = @timed threaded_sleep() + # if we have at least 2 threads, the sleeps run in parallel and take about a second + if Threads.nthreads() ≥ 2 + @test 0.9 < t.time < 1.2 + end +end + +@testitem "@release and @lock" begin + # This calls Python's time.sleep(1) twice concurrently. Since sleep() releases the + # GIL, these can happen in parallel if Julia has at least 2 threads. + function threaded_sleep() + PythonCall.GIL.@release Threads.@threads for i = 1:2 + PythonCall.GIL.@lock pyimport("time").sleep(1) + end + end + # one run to ensure it's compiled + threaded_sleep() + # now time it + t = @timed threaded_sleep() + # if we have at least 2 threads, the sleeps run in parallel and take about a second + if Threads.nthreads() ≥ 2 + @test 0.9 < t.time < 1.2 + end +end From a183de1cb529efe27df2b9f6a2efb92d5ea40100 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 21:04:14 +0100 Subject: [PATCH 21/37] add GIL to release notes --- docs/src/releasenotes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 33141da8..21ff4b2b 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -7,6 +7,9 @@ * `GC.disable()` and `GC.enable()` are now a no-op and deprecated since they are no longer required for thread-safety. These will be removed in v1. * Adds `GC.gc()`. +* Adds module `PythonCall.GIL` with `lock()`, `release()`, `@lock` and `@release` for + handling the Python Global Interpreter Lock. In combination with the above + improvements, these allow Julia and Python to co-operate on multiple threads. ## 0.9.21 (2024-07-20) * `Serialization.serialize` can use `dill` instead of `pickle` by setting the env var `JULIA_PYTHONCALL_PICKLE=dill`. From b75f0a1bd391bcb951a0c9fc0367b31de347a2df Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 21:04:40 +0100 Subject: [PATCH 22/37] add GIL release tests from Python --- pytest/test_all.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/pytest/test_all.py b/pytest/test_all.py index f94f91f6..0b5a6005 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -100,3 +100,68 @@ def test_julia_gc(): @test isempty(PythonCall.GC.QUEUE.items) """ ) + +def test_gil_release(): + """Tests that we can execute Julia code in parallel by releasing the GIL.""" + from concurrent.futures import ThreadPoolExecutor, wait + from time import time + from juliacall import Main as jl + # julia implementation of sleep which releases the GIL + # this test uses Base.Libc.systemsleep which does not yield to the scheduler + jsleep = jl.seval("t::Real -> PythonCall.GIL.@release Libc.systemsleep(t)") + # precompile + jsleep(0.01) + # use two threads + pool = ThreadPoolExecutor(2) + # run sleep twice concurrently + t0 = time() + fs = [pool.submit(jsleep, 1) for _ in range(2)] + t1 = time() - t0 + wait(fs) + t2 = time() - t0 + # submitting tasks should be very fast + assert t1 < 0.1 + # executing the tasks should take about 1 second because they happen in parallel + assert 0.9 < t2 < 1.5 + +def test_gil_release_2(): + """Same as the previous test but with a function (sleep) that yields. + + Yielding puts us back into Python, which itself doesn't ever yield back to Julia, so + the function can never return. Hence for the threads to finish, we need to + explicitly yield back to Julia. + """ + from concurrent.futures import ThreadPoolExecutor, wait + from time import sleep, time + from juliacall import Main as jl + # julia implementation of sleep which releases the GIL + # in this test we use Base.sleep which yields to the scheduler + jsleep = jl.seval("t::Real -> PythonCall.GIL.@release sleep(t)") + jyield = jl.seval("yield") + # precompile + jsleep(0.01) + jyield() + # use two threads + pool = ThreadPoolExecutor(2) + # run sleep twice concurrently + t0 = time() + fs = [pool.submit(jsleep, 1) for _ in range(2)] + t1 = time() - t0 + # because sleep() yields to the scheduler, which puts us back in Python, we need to + # explicitly yield back to give the scheduler a chance to finish the sleep calls, so + # we yield every 0.1 seconds + done = False + for _ in range(20): + if any(f.running() for f in fs): + sleep(0.1) + jyield() + else: + done = True + break + t2 = time() - t0 + # submitting tasks should be very fast + assert t1 < 0.1 + # the tasks should have finished + assert done + # executing the tasks should take about 1 second because they happen in parallel + assert 0.9 < t2 < 1.5 From 31cd57da9db661da87339ee346c21bee454ddbe5 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 21:05:36 +0100 Subject: [PATCH 23/37] typo: testset -> testitem --- test/GC.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/GC.jl b/test/GC.jl index 93454100..f53bfec5 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1,4 +1,4 @@ -@testset "201: GC segfaults" begin +@testitem "201: GC segfaults" begin # https://github.com/JuliaPy/PythonCall.jl/issues/201 # This should not segfault! cmd = Base.julia_cmd() @@ -7,7 +7,7 @@ @test p.exitcode == 0 end -@testset "GC.gc()" begin +@testitem "GC.gc()" begin let pyobjs = map(pylist, 1:100) Threads.@threads for obj in pyobjs @@ -19,7 +19,7 @@ end @test isempty(PythonCall.GC.QUEUE.items) end -@testset "GC.GCHook" begin +@testitem "GC.GCHook" begin let pyobjs = map(pylist, 1:100) Threads.@threads for obj in pyobjs From 73f7eb8fa1d96c98434d60ddf50f54ddcd031a7e Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 21:06:03 +0100 Subject: [PATCH 24/37] delete redundant test --- test/GC.jl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/GC.jl b/test/GC.jl index f53bfec5..3691a315 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1,12 +1,3 @@ -@testitem "201: GC segfaults" begin - # https://github.com/JuliaPy/PythonCall.jl/issues/201 - # This should not segfault! - cmd = Base.julia_cmd() - path = joinpath(@__DIR__, "finalize_test_script.jl") - p = run(`$cmd -t2 --project $path`) - @test p.exitcode == 0 -end - @testitem "GC.gc()" begin let pyobjs = map(pylist, 1:100) From 2a54ca9b85e0cc8280b58f162dce0b7968c5ca9c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 21:20:43 +0100 Subject: [PATCH 25/37] remove out of date comment --- pytest/test_all.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytest/test_all.py b/pytest/test_all.py index f94f91f6..10f78462 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -81,8 +81,6 @@ def test_julia_gc(): # We make a bunch of python objects with no reference to them, # then call GC to try to finalize them. # We want to make sure we don't segfault. - # Here we can (manually) verify that the background task is running successfully, - # by seeing the printout "Python GC (100 items): 0.000000 seconds." # We also programmatically check things are working by verifying the queue is empty. # Debugging note: if you get segfaults, then run the tests with # `PYTHON_JULIACALL_HANDLE_SIGNALS=yes python3 -X faulthandler -m pytest -p no:faulthandler -s --nbval --cov=pysrc ./pytest/` From ca64d21ff2af861db344d9736f28f8666ab5af9d Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 21:40:53 +0100 Subject: [PATCH 26/37] comment erroneous test --- test/GC.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/GC.jl b/test/GC.jl index 3691a315..2467f694 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -5,7 +5,9 @@ finalize(obj) end end - Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + # The GC sometimes actually frees everything before this line. + # We can uncomment this line if we GIL.@release the above block once we have it. + # Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end @@ -17,7 +19,9 @@ end finalize(obj) end end - Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + # The GC sometimes actually frees everything before this line. + # We can uncomment this line if we GIL.@release the above block once we have it. + # Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end From 73e98223d55fbab8c72ec296b34813c70e30f74c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 2 Aug 2024 21:44:53 +0100 Subject: [PATCH 27/37] re-enable commented test --- test/GC.jl | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test/GC.jl b/test/GC.jl index 2467f694..cc8d6ff0 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1,13 +1,11 @@ @testitem "GC.gc()" begin let pyobjs = map(pylist, 1:100) - Threads.@threads for obj in pyobjs + PythonCall.GIL.@release Threads.@threads for obj in pyobjs finalize(obj) end end - # The GC sometimes actually frees everything before this line. - # We can uncomment this line if we GIL.@release the above block once we have it. - # Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end @@ -15,13 +13,11 @@ end @testitem "GC.GCHook" begin let pyobjs = map(pylist, 1:100) - Threads.@threads for obj in pyobjs + PythonCall.GIL.@release Threads.@threads for obj in pyobjs finalize(obj) end end - # The GC sometimes actually frees everything before this line. - # We can uncomment this line if we GIL.@release the above block once we have it. - # Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end From 1f67135743d4de2db9328f1439d2c8b4189a8624 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 10:50:30 +0100 Subject: [PATCH 28/37] adds AnyValue._jl_call_nogil --- pytest/test_all.py | 25 +++++++++++-------------- src/JlWrap/JlWrap.jl | 1 + src/JlWrap/any.jl | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/pytest/test_all.py b/pytest/test_all.py index 3074feb9..eca0d3d5 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -99,14 +99,14 @@ def test_julia_gc(): """ ) -def test_gil_release(): +def test_call_nogil(): """Tests that we can execute Julia code in parallel by releasing the GIL.""" from concurrent.futures import ThreadPoolExecutor, wait from time import time from juliacall import Main as jl # julia implementation of sleep which releases the GIL # this test uses Base.Libc.systemsleep which does not yield to the scheduler - jsleep = jl.seval("t::Real -> PythonCall.GIL.@release Libc.systemsleep(t)") + jsleep = jl.Libc.systemsleep._jl_call_nogil # precompile jsleep(0.01) # use two threads @@ -122,7 +122,7 @@ def test_gil_release(): # executing the tasks should take about 1 second because they happen in parallel assert 0.9 < t2 < 1.5 -def test_gil_release_2(): +def test_call_nogil_yielding(): """Same as the previous test but with a function (sleep) that yields. Yielding puts us back into Python, which itself doesn't ever yield back to Julia, so @@ -134,8 +134,8 @@ def test_gil_release_2(): from juliacall import Main as jl # julia implementation of sleep which releases the GIL # in this test we use Base.sleep which yields to the scheduler - jsleep = jl.seval("t::Real -> PythonCall.GIL.@release sleep(t)") - jyield = jl.seval("yield") + jsleep = jl.sleep._jl_call_nogil + jyield = getattr(jl, "yield") # precompile jsleep(0.01) jyield() @@ -148,18 +148,15 @@ def test_gil_release_2(): # because sleep() yields to the scheduler, which puts us back in Python, we need to # explicitly yield back to give the scheduler a chance to finish the sleep calls, so # we yield every 0.1 seconds - done = False - for _ in range(20): - if any(f.running() for f in fs): - sleep(0.1) - jyield() - else: - done = True - break + status = wait(fs, timeout=0.1) t2 = time() - t0 + while t2 < 2.0 and status.not_done: + jyield() + status = wait(fs, timeout=0.1) + t2 = time() - t0 # submitting tasks should be very fast assert t1 < 0.1 # the tasks should have finished - assert done + assert not status.not_done # executing the tasks should take about 1 second because they happen in parallel assert 0.9 < t2 < 1.5 diff --git a/src/JlWrap/JlWrap.jl b/src/JlWrap/JlWrap.jl index 2a55123a..637018da 100644 --- a/src/JlWrap/JlWrap.jl +++ b/src/JlWrap/JlWrap.jl @@ -42,6 +42,7 @@ using ..Convert: pyconvertarg, pyconvert_result using ..GC: GC +using ..GIL: GIL using Pkg: Pkg using Base: @propagate_inbounds, allocatedinline diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index d5cad426..0e91e7f1 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -51,6 +51,24 @@ end pyjl_handle_error_type(::typeof(pyjlany_call), self, exc) = exc isa MethodError && exc.f === self ? pybuiltins.TypeError : PyNULL +function pyjlany_call_nogil(self, args_::Py, kwargs_::Py) + if pylen(kwargs_) > 0 + args = pyconvert(Vector{Any}, args_) + kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) + ans = Py(GIL.@release self(args...; kwargs...)) + elseif pylen(args_) > 0 + args = pyconvert(Vector{Any}, args_) + ans = Py(GIL.@release self(args...)) + else + ans = Py(GIL.@release self()) + end + pydel!(args_) + pydel!(kwargs_) + ans +end +pyjl_handle_error_type(::typeof(pyjlany_call_nogil), self, exc) = + exc isa MethodError && exc.f === self ? pybuiltins.TypeError : PyNULL + function pyjlany_getitem(self, k_::Py) if pyistuple(k_) k = pyconvert(Vector{Any}, k_) @@ -339,6 +357,8 @@ class AnyValue(ValueBase): return self._jl_callmethod($(pyjl_methodnum(pyjlany_display)), mime) def _jl_help(self, mime=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_help)), mime) + def _jl_call_nogil(self, *args, **kwargs): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_call_nogil)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) """, From 276e80f8c4b6389c90adf6a4351619b0ff289cfc Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 11:05:58 +0100 Subject: [PATCH 29/37] add RawValue._jl_call_nogil --- pytest/test_all.py | 92 ++++++++++++++++++++++++---------------------- src/JlWrap/raw.jl | 18 +++++++++ 2 files changed, 66 insertions(+), 44 deletions(-) diff --git a/pytest/test_all.py b/pytest/test_all.py index eca0d3d5..c7adac5b 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -1,29 +1,40 @@ +import pytest + + def test_import(): import juliacall + def test_newmodule(): import juliacall + jl = juliacall.Main m = juliacall.newmodule("TestModule") assert isinstance(m, juliacall.ModuleValue) assert jl.isa(m, jl.Module) assert str(jl.nameof(m)) == "TestModule" + def test_convert(): import juliacall + jl = juliacall.Main - for (x, t) in [(None, jl.Nothing), (True, jl.Bool), ([1,2,3], jl.Vector)]: + for x, t in [(None, jl.Nothing), (True, jl.Bool), ([1, 2, 3], jl.Vector)]: y = juliacall.convert(t, x) assert isinstance(y, juliacall.AnyValue) assert jl.isa(y, t) + def test_interactive(): import juliacall + juliacall.interactive(True) juliacall.interactive(False) + def test_JuliaError(): import juliacall + jl = juliacall.Main assert isinstance(juliacall.JuliaError, type) assert issubclass(juliacall.JuliaError, Exception) @@ -40,11 +51,13 @@ def test_JuliaError(): bt = err.backtrace assert bt is not None + def test_issue_394(): "https://github.com/JuliaPy/PythonCall.jl/issues/394" from juliacall import Main as jl + x = 3 - f = lambda x: x+1 + f = lambda x: x + 1 y = 5 jl.x = x assert jl.x is x @@ -57,6 +70,7 @@ def test_issue_394(): assert jl.y is y assert jl.seval("f(x)") == 4 + def test_issue_433(): "https://github.com/JuliaPy/PythonCall.jl/issues/433" from juliacall import Main as jl @@ -76,8 +90,10 @@ def test_issue_433(): ) assert out == 25 + def test_julia_gc(): from juliacall import Main as jl + # We make a bunch of python objects with no reference to them, # then call GC to try to finalize them. # We want to make sure we don't segfault. @@ -99,64 +115,52 @@ def test_julia_gc(): """ ) -def test_call_nogil(): + +@pytest.mark.parametrize( + ["yld", "raw"], [(yld, raw) for yld in [False, True] for raw in [False, True]] +) +def test_call_nogil(yld, raw): """Tests that we can execute Julia code in parallel by releasing the GIL.""" from concurrent.futures import ThreadPoolExecutor, wait from time import time from juliacall import Main as jl - # julia implementation of sleep which releases the GIL - # this test uses Base.Libc.systemsleep which does not yield to the scheduler - jsleep = jl.Libc.systemsleep._jl_call_nogil - # precompile - jsleep(0.01) - # use two threads - pool = ThreadPoolExecutor(2) - # run sleep twice concurrently - t0 = time() - fs = [pool.submit(jsleep, 1) for _ in range(2)] - t1 = time() - t0 - wait(fs) - t2 = time() - t0 - # submitting tasks should be very fast - assert t1 < 0.1 - # executing the tasks should take about 1 second because they happen in parallel - assert 0.9 < t2 < 1.5 -def test_call_nogil_yielding(): - """Same as the previous test but with a function (sleep) that yields. - - Yielding puts us back into Python, which itself doesn't ever yield back to Julia, so - the function can never return. Hence for the threads to finish, we need to - explicitly yield back to Julia. - """ - from concurrent.futures import ThreadPoolExecutor, wait - from time import sleep, time - from juliacall import Main as jl # julia implementation of sleep which releases the GIL - # in this test we use Base.sleep which yields to the scheduler - jsleep = jl.sleep._jl_call_nogil + if yld: + # use sleep, which yields + jsleep = jl.sleep + else: + # use Libc.systemsleep which does not yield + jsleep = jl.Libc.systemsleep + if raw: + # test RawValue instead of AnyValue + jsleep = jsleep._jl_raw() + jsleep = jsleep._jl_call_nogil jyield = getattr(jl, "yield") # precompile jsleep(0.01) jyield() # use two threads pool = ThreadPoolExecutor(2) - # run sleep twice concurrently + # run jsleep(1) twice concurrently t0 = time() fs = [pool.submit(jsleep, 1) for _ in range(2)] + # submitting tasks should be very fast t1 = time() - t0 - # because sleep() yields to the scheduler, which puts us back in Python, we need to - # explicitly yield back to give the scheduler a chance to finish the sleep calls, so - # we yield every 0.1 seconds - status = wait(fs, timeout=0.1) - t2 = time() - t0 - while t2 < 2.0 and status.not_done: - jyield() + assert t1 < 0.1 + # wait for the tasks to finish + if yld: + # we need to explicitly yield back to give the Julia scheduler a chance to + # finish the sleep calls, so we yield every 0.1 seconds status = wait(fs, timeout=0.1) t2 = time() - t0 - # submitting tasks should be very fast - assert t1 < 0.1 - # the tasks should have finished - assert not status.not_done + while status.not_done: + jyield() + status = wait(fs, timeout=0.1) + t2 = time() - t0 + assert t2 < 2.0 + else: + wait(fs) + t2 = time() - t0 # executing the tasks should take about 1 second because they happen in parallel assert 0.9 < t2 < 1.5 diff --git a/src/JlWrap/raw.jl b/src/JlWrap/raw.jl index 26456b21..26b893b9 100644 --- a/src/JlWrap/raw.jl +++ b/src/JlWrap/raw.jl @@ -40,6 +40,22 @@ function pyjlraw_call(self, args_::Py, kwargs_::Py) ans end +function pyjlraw_call_nogil(self, args_::Py, kwargs_::Py) + if pylen(kwargs_) > 0 + args = pyconvert(Vector{Any}, args_) + kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) + ans = pyjlraw(GIL.@release self(args...; kwargs...)) + elseif pylen(args_) > 0 + args = pyconvert(Vector{Any}, args_) + ans = pyjlraw(GIL.@release self(args...)) + else + ans = pyjlraw(GIL.@release self()) + end + pydel!(args_) + pydel!(kwargs_) + ans +end + pyjlraw_len(self) = Py(length(self)) function pyjlraw_getitem(self, k_::Py) @@ -130,6 +146,8 @@ class RawValue(ValueBase): return self._jl_callmethod($(pyjl_methodnum(pyjlraw_bool))) def _jl_any(self): return self._jl_callmethod($(pyjl_methodnum(pyjl))) + def _jl_call_nogil(self, *args, **kwargs): + return self._jl_callmethod($(pyjl_methodnum(pyjlraw_call_nogil)), args, kwargs) """, @__FILE__(), "exec", From dfdf8d1668987d57c48e7cae0b6137e4dbed0c95 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 11:12:12 +0100 Subject: [PATCH 30/37] add docstrings --- docs/src/juliacall-reference.md | 6 ++++-- src/JlWrap/any.jl | 4 ++++ src/JlWrap/raw.jl | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/src/juliacall-reference.md b/docs/src/juliacall-reference.md index c62a480b..b9b4d485 100644 --- a/docs/src/juliacall-reference.md +++ b/docs/src/juliacall-reference.md @@ -93,8 +93,9 @@ replaced with `!!`. ###### Members - `_jl_raw()`: Convert to a [`RawValue`](#juliacall.RawValue). (See also [`pyjlraw`](@ref).) -- `_jl_display()`: Display the object using Julia's display mechanism. -- `_jl_help()`: Display help for the object. +- `_jl_display(mime=None)`: Display the object using Julia's display mechanism. +- `_jl_help(mime=None)`: Display help for the object. +- `_jl_call_nogil(*args, **kwargs)`: Call this with the GIL disabled. ````` `````@customdoc @@ -217,4 +218,5 @@ single tuple, it will need to be wrapped in another tuple. ###### Members - `_jl_any()`: Convert to a [`AnyValue`](#juliacall.AnyValue) (or subclass). (See also [`pyjl`](@ref).) +- `_jl_call_nogil(*args, **kwargs)`: Call this with the GIL disabled. ````` diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index 0e91e7f1..e3f7b5d5 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -352,12 +352,16 @@ class AnyValue(ValueBase): def __name__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_name))) def _jl_raw(self): + '''Convert this to a juliacall.RawValue.''' return self._jl_callmethod($(pyjl_methodnum(pyjlraw))) def _jl_display(self, mime=None): + '''Display this, optionally specifying the MIME type.''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_display)), mime) def _jl_help(self, mime=None): + '''Show help for this Julia object.''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_help)), mime) def _jl_call_nogil(self, *args, **kwargs): + '''Call this with the given arguments but with the GIL disabled.''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_call_nogil)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) diff --git a/src/JlWrap/raw.jl b/src/JlWrap/raw.jl index 26b893b9..f5446601 100644 --- a/src/JlWrap/raw.jl +++ b/src/JlWrap/raw.jl @@ -145,8 +145,10 @@ class RawValue(ValueBase): def __bool__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlraw_bool))) def _jl_any(self): + '''Convert this to a juliacall.AnyValue.''' return self._jl_callmethod($(pyjl_methodnum(pyjl))) def _jl_call_nogil(self, *args, **kwargs): + '''Call this with the given arguments but with the GIL disabled.''' return self._jl_callmethod($(pyjl_methodnum(pyjlraw_call_nogil)), args, kwargs) """, @__FILE__(), From 2519610d68257280b23fc17ce0d2bbc2ab08f87b Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 11:16:56 +0100 Subject: [PATCH 31/37] add warnings about the GIL to docstrings --- src/JlWrap/any.jl | 6 +++++- src/JlWrap/raw.jl | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index e3f7b5d5..573ca1a4 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -361,7 +361,11 @@ class AnyValue(ValueBase): '''Show help for this Julia object.''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_help)), mime) def _jl_call_nogil(self, *args, **kwargs): - '''Call this with the given arguments but with the GIL disabled.''' + '''Call this with the given arguments but with the GIL disabled. + + WARNING: This function must not interact with Python at all without re-acquiring + the GIL. + ''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_call_nogil)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) diff --git a/src/JlWrap/raw.jl b/src/JlWrap/raw.jl index f5446601..21e0fa7e 100644 --- a/src/JlWrap/raw.jl +++ b/src/JlWrap/raw.jl @@ -148,7 +148,11 @@ class RawValue(ValueBase): '''Convert this to a juliacall.AnyValue.''' return self._jl_callmethod($(pyjl_methodnum(pyjl))) def _jl_call_nogil(self, *args, **kwargs): - '''Call this with the given arguments but with the GIL disabled.''' + '''Call this with the given arguments but with the GIL disabled. + + WARNING: This function must not interact with Python at all without re-acquiring + the GIL. + ''' return self._jl_callmethod($(pyjl_methodnum(pyjlraw_call_nogil)), args, kwargs) """, @__FILE__(), From 0d02000fe99a413307f30e4834093dbd07c2252c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 12:32:28 +0100 Subject: [PATCH 32/37] add reference docstrings --- docs/src/juliacall-reference.md | 2 +- docs/src/pythoncall-reference.md | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/src/juliacall-reference.md b/docs/src/juliacall-reference.md index b9b4d485..b67d8b26 100644 --- a/docs/src/juliacall-reference.md +++ b/docs/src/juliacall-reference.md @@ -1,4 +1,4 @@ -# JuliaCall API Reference +# [JuliaCall API Reference](@id jl-reference) ## Constants diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 75065f50..018cfdfe 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -218,6 +218,19 @@ Py(x::MyType) = x.py @pyconst ``` +## Multi-threading + +These functions are not exported. They support multi-threading of Python and/or Julia. +See also [`juliacall.AnyValue._jl_call_nogil`](@ref julia-wrappers). + +```@docs +PythonCall.GIL.lock +PythonCall.GIL.@lock +PythonCall.GIL.release +PythonCall.GIL.@release +PythonCall.GC.gc +``` + ## The Python interpreter These functions are not exported. They give information about which Python interpreter is From f035fa3c982269e57f6137e8e2e2397c5727e9e2 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 12:42:51 +0100 Subject: [PATCH 33/37] remove big pycall comparison and move pycall help to faq --- README.md | 5 ++-- docs/make.jl | 1 - docs/src/faq.md | 10 +++++++ docs/src/pycall.md | 75 ---------------------------------------------- 4 files changed, 12 insertions(+), 79 deletions(-) delete mode 100644 docs/src/pycall.md diff --git a/README.md b/README.md index fccc0db9..d5cc60db 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,8 @@ In this example we use the Python module JuliaCall from an IPython notebook to t ## What about PyCall? -The existing package [PyCall](https://github.com/JuliaPy/PyCall.jl) is another similar interface to Python. Here we note some key differences, but a more detailed comparison is in the documentation. +The existing package [PyCall](https://github.com/JuliaPy/PyCall.jl) is another similar interface to Python. Here we note some key differences:. - PythonCall supports a wider range of conversions between Julia and Python, and the conversion mechanism is extensible. - PythonCall by default never copies mutable objects when converting, but instead directly wraps the mutable object. This means that modifying the converted object modifies the original, and conversion is faster. - PythonCall does not usually automatically convert results to Julia values, but leaves them as Python objects. This makes it easier to do Pythonic things with these objects (e.g. accessing methods) and is type-stable. -- PythonCall installs dependencies into a separate Conda environment for each Julia project. This means each Julia project can have an isolated set of Python dependencies. -- PythonCall supports Julia 1.6.1+ and Python 3.8+ whereas PyCall supports Julia 0.7+ and Python 2.7+. +- PythonCall installs dependencies into a separate Conda environment for each Julia project using [CondaPkg](https://github.com/JuliaPy/CondaPkg.jl). This means each Julia project can have an isolated set of Python dependencies. diff --git a/docs/make.jl b/docs/make.jl index 605d204d..7b04641e 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -19,7 +19,6 @@ makedocs( ], "compat.md", "faq.md", - "pycall.md", "releasenotes.md", ], ) diff --git a/docs/src/faq.md b/docs/src/faq.md index b51717d0..cfce97c1 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -1,5 +1,15 @@ # FAQ & Troubleshooting +## Can I use PythonCall and PyCall together? + +Yes, you can use both PyCall and PythonCall in the same Julia session. This is platform-dependent: +- On most systems the Python interpreter used by PythonCall and PyCall must be the same (see below). +- On Windows it appears to be possible for PythonCall and PyCall to use different interpreters. + +To force PythonCall to use the same Python interpreter as PyCall, set the environment variable [`JULIA_PYTHONCALL_EXE`](@ref pythoncall-config) to `"@PyCall"`. Note that this will opt out of automatic dependency management using CondaPkg. + +Alternatively, to force PyCall to use the same interpreter as PythonCall, set the environment variable `PYTHON` to [`PythonCall.python_executable_path()`](@ref) and then `Pkg.build("PyCall")`. You will need to do this each time you change project, because PythonCall by default uses a different Python for each project. + ## Is PythonCall/JuliaCall thread safe? No. diff --git a/docs/src/pycall.md b/docs/src/pycall.md deleted file mode 100644 index e96f89dd..00000000 --- a/docs/src/pycall.md +++ /dev/null @@ -1,75 +0,0 @@ -# Coming from PyCall/PyJulia? - -Another similar interface to Python is provided by [PyCall](https://github.com/JuliaPy/PyCall.jl). - -On this page, we give some tips for migrating between the two modules and a comparison. - -## Tips - -- You can use both PyCall and PythonCall in the same Julia session. This is platform-dependent: - - On Unix (Linux, Mac, etc.) the Python interpreter used by PythonCall and PyCall must be the same (see below). - - On Windows, it appears to be possible for PythonCall and PyCall to use different interpreters. -- To force PythonCall to use the same Python interpreter as PyCall, set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. Note that this will opt out of automatic dependency management using CondaPkg. -- Alternatively, to force PyCall to use the same interpreter as PythonCall, set the environment variable `PYTHON` to `PythonCall.python_executable_path()` and then `Pkg.build("PyCall")`. You will need to do this each time you change project, because PythonCall by default uses a different Python for each project. - -## Comparison - -### Flexibility of conversion - -In PyCall you do `convert(T, x)` to convert the Python object `x` to a Julia `T`. In PythonCall you similarly do `pyconvert(T, x)`. - -PythonCall supports far more combinations of types of `T` and `x`. For example `convert(Vector, x)` in PyCall requires `x` to be a sequence, whereas in PythonCall `pyconvert(Vector, x)` works if `x` is an iterable, an object supporting the buffer protocol (such as `bytes`) or an object supporting the numpy array interface (such as `numpy.ndarray`). - -Furthermore, `pyconvert` can be extended to support more types, whereas `convert(Vector, x)` cannot support more Python types. - -### Lossiness of conversion - -Both packages allow conversion of Julia values to Python: `PyObject(x)` in PyCall, `Py(x)` in PythonCall. - -Whereas both packages convert numbers, booleans, tuples and strings to their Python counterparts, they differ in handling other types. For example PyCall converts `AbstractVector` to `list` whereas PythonCall converts `AbstractVector` to `juliacall.VectorValue` which is a sequence type directly wrapping the Julia value - this has the advantage that mutating the Python object also mutates the original Julia object. - -Hence with PyCall the following does not mutate the original array `x`: -```julia -x = ["foo", "bar"] -PyObject(x).append("baz") -@show x # --> ["foo", "bar"] -``` -whereas with PythonCall the following does mutate `x`: -```julia -x = ["foo", "bar"] -Py(x).append("baz") -@show x # --> ["foo", "bar", "baz"] -``` - -In fact, PythonCall has the policy that any mutable object will by default be wrapped in this way, which not only preserves mutability but makes conversion faster for large containers since it does not require taking a copy of all the data. - -### Automatic conversion - -In PyCall, most function calls, attribute accesses, indexing, etc. of Python object by default automatically convert their result to a Julia object. This means that the following -```julia -pyimport("sys").modules["KEY"] = "VALUE" -``` -does not actually modify the modules dict because it was *copied* to a new Julia `Dict`. This was probably not intended, plus it wasted time copying the whole dictionary. Instead you must do -```julia -set!(pyimport(os)."environ", "KEY", "VALUE") -``` - -In PythonCall, we don't do any such automatic conversion: we always return `Py`. This means that the first piece of code above does what you think. - -### Which Python - -PyCall uses some global installation of Python - typically the version of Python installed on the system or used by Conda. - -PythonCall uses a separate Conda environment for each Julia environment/project/package and installs Python (and other Python packages) into that. This means that different Julia projects can maintain an isolated set of Python dependencies (including the Python version itself). - -### Corresponding Python packages - -PyCall has the corresponding Python package [PyJulia](https://github.com/JuliaPy/pyjulia) for calling Julia from Python, and PythonCall similarly has JuliaCall. - -One difference is between them is their code size: PyJulia is a large package, whereas JuliaCall is very small, with most of the implementation being in PythonCall itself. The practical up-shot is that PythonCall/JuliaCall have very symmetric interfaces; for example they use identical conversion policies and have the same set of wrapper types available. - -Note also that JuliaCall will use a separate Julia project for each virtual/conda environment. This means that different Python environments can maintain an isolated set of Julia dependencies, including the versions of Julia and PythonCall themselves. - -### Compatibility - -PyCall supports Julia 0.7+ and Python 2.7+, whereas PythonCall supports Julia 1.6.1+ and Python 3.8+. PyCall requires numpy to be installed, PythonCall doesn't (it provides the same fast array access through the buffer protocol and array interface). From 660b1e6080d3898ff82928b752f94fdd37b065f5 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 13:52:47 +0100 Subject: [PATCH 34/37] document new threading features --- docs/src/faq.md | 19 +++-------- docs/src/juliacall.md | 76 ++++++++++++++++++++++++++++++++++++++++++ docs/src/pythoncall.md | 40 ++++++++++++++++++++++ 3 files changed, 121 insertions(+), 14 deletions(-) diff --git a/docs/src/faq.md b/docs/src/faq.md index cfce97c1..eefa12df 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -12,20 +12,11 @@ Alternatively, to force PyCall to use the same interpreter as PythonCall, set th ## Is PythonCall/JuliaCall thread safe? -No. - -However it is safe to use PythonCall with Julia with multiple threads, provided you only -call Python code from the first thread. (Before v0.9.22, tricks such as disabling the -garbage collector were required.) - -From Python, to use JuliaCall with multiple threads you probably need to set -[`PYTHON_JULIACALL_HANDLE_SIGNALS=yes`](@ref julia-config) before importing JuliaCall. -This is because Julia intentionally causes segmentation faults as part of the GC -safepoint mechanism. If unhandled, these segfaults will result in termination of the -process. This is equivalent to starting julia with `julia --handle-signals=yes`, the -default behavior in Julia. See discussion -[here](https://github.com/JuliaPy/PythonCall.jl/issues/219#issuecomment-1605087024) -for more information. +Yes, as of v0.9.22, provided you handle the GIL correctly. See the guides for +[PythonCall](@ref jl-multi-threading) and [JuliaCall](@ref py-multi-threading). + +Before, tricks such as disabling the garbage collector were required. See the +[old docs](https://juliapy.github.io/PythonCall.jl/v0.9.21/faq/#Is-PythonCall/JuliaCall-thread-safe?). Related issues: [#201](https://github.com/JuliaPy/PythonCall.jl/issues/201), diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index a16b71f2..9d46ac57 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -124,3 +124,79 @@ be configured in two ways: | `-X juliacall-threads=` | `PYTHON_JULIACALL_THREADS=` | Launch N threads. | | `-X juliacall-warn-overwrite=` | `PYTHON_JULIACALL_WARN_OVERWRITE=` | Enable or disable method overwrite warnings. | | `-X juliacall-autoload-ipython-extension=` | `PYTHON_JULIACALL_AUTOLOAD_IPYTHON_EXTENSION=` | Enable or disable IPython extension autoloading. | + +## [Multi-threading](@id py-multi-threading) + +From v0.9.22, JuliaCall supports multi-threading in Julia and/or Python, with some +caveats. + +Most importantly, you can only call Python code while Python's +[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) +is held by the current thread. You can use JuliaCall from any Python thread, and the GIL +will be held whenever any JuliaCall function is used. However, to leverage the benefits +of multi-threading, you can release the GIL while executing any Julia code that does not +interact with Python. + +The simplest way to do this is using the `_jl_call_nogil` method on Julia functions to +call the function with the GIL released. + +```python +from concurrent.futures import ThreadPoolExecutor, wait +from juliacall import Main as jl +pool = ThreadPoolExecutor(4) +fs = [pool.submit(jl.Libc.systemsleep._jl_call_nogil, 5) for _ in range(4)] +wait(fs) +``` + +In the above example, we call `Libc.systemsleep(5)` on four threads. Because we +called it with `_jl_call_nogil`, the GIL was released, allowing the threads to run in +parallel, taking about 5 seconds in total. + +If we did not use `_jl_call_nogil` (i.e. if we did `pool.submit(jl.Libc.systemsleep, 5)`) +then the above code will take 20 seconds because the sleeps run one after another. + +It is very important that any function called with `_jl_call_nogil` does not interact +with Python at all unless it re-acquires the GIL first, such as by using +[PythonCall.GIL.@lock](@ref). + +You can also use [multi-threading from Julia](@ref jl-multi-threading). + +### Caveat: Julia's task scheduler + +If you try the above example with a Julia function that yields to the task scheduler, +such as `sleep` instead of `Libc.systemsleep`, then you will likely experience a hang. + +In this case, you need to yield back to Julia's scheduler periodically to allow the task +to continue. You can use the following pattern instead of `wait(fs)`: +```python +jl_yield = getattr(jl, "yield") +while True: + # yield to Julia's task scheduler + jl_yield() + # wait for up to 0.1 seconds for the threads to finish + state = wait(fs, timeout=0.1) + # if they finished then stop otherwise try again + if not state.not_done: + break +``` + +Set the `timeout` parameter smaller to let Julia's scheduler cycle more frequently. + +Future versions of JuliaCall may provide tooling to make this simpler. + +### [Caveat: Signal handling](@id py-multi-threading-signal-handling) + +We recommend setting [`PYTHON_JULIACALL_HANDLE_SIGNALS=yes`](@ref julia-config) +before importing JuliaCall with multiple threads. + +This is because Julia intentionally causes segmentation faults as part of the GC +safepoint mechanism. If unhandled, these segfaults will result in termination of the +process. See discussion +[here](https://github.com/JuliaPy/PythonCall.jl/issues/219#issuecomment-1605087024) +for more information. + +Note however that this interferes with Python's own signal handling, so for example +Ctrl-C will not raise `KeyboardInterrupt`. + +Future versions of JuliaCall may make this the default behaviour when using multiple +threads. diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index a69b3b46..1c5f3c0c 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -362,3 +362,43 @@ end If your package depends on some Python packages, you must generate a `CondaPkg.toml` file. See [Installing Python packages](@ref python-deps). + +## [Multi-threading](@id jl-multi-threading) + +From v0.9.22, PythonCall supports multi-threading in Julia and/or Python, with some +caveats. + +Most importantly, you can only call Python code while Python's +[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) +is held by the current thread. Ordinarily, the GIL is held by the main thread in Julia, +so if you want to run Python code on any other thread, you must release the GIL from the +main thread and then re-acquire it while running any Python code on other threads. + +This is made possible by the macros [`PythonCall.GIL.@release`](@ref) and +[`PythonCall.GIL.@lock`](@ref) or the functions [`PythonCall.GIL.release`](@ref) and +[`PythonCall.GIL.lock`](@ref) with this pattern: + +```julia +PythonCall.GIL.@release Threads.@threads for i in 1:4 + PythonCall.GIL.@lock pyimport("time").sleep(5) +end +``` + +In the above example, we call `time.sleep(5)` four times in parallel. If Julia was +started with at least four threads (`julia -t4`) then the above code will take about +5 seconds. + +Both `@release` and `@lock` are important. If the GIL were not released, then a deadlock +would occur when attempting to lock the already-locked GIL from the threads. If the GIL +were not re-acquired, then Python would crash when interacting with it. + +You can also use [multi-threading from Python](@ref py-multi-threading). + +### Caveat: Garbage collection + +If Julia's GC collects any Python objects from a thread where the GIL is not currently +held, then those Python objects will not immediately be deleted. Instead they will be +queued to be released in a later GC pass. + +If you find you have many Python objects not being deleted, you can call +[`PythonCall.GC.gc()`](@ref) or `GC.gc()` (while the GIL is held) to clear the queue. From 7d2f04ef063a48a4eb9a449db8ef870fa950e6d3 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 13:55:08 +0100 Subject: [PATCH 35/37] update release notes --- docs/src/releasenotes.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 21ff4b2b..f5d5eaef 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -7,9 +7,11 @@ * `GC.disable()` and `GC.enable()` are now a no-op and deprecated since they are no longer required for thread-safety. These will be removed in v1. * Adds `GC.gc()`. -* Adds module `PythonCall.GIL` with `lock()`, `release()`, `@lock` and `@release` for - handling the Python Global Interpreter Lock. In combination with the above - improvements, these allow Julia and Python to co-operate on multiple threads. +* Adds module `GIL` with `lock()`, `release()`, `@lock` and `@release` for handling the + Python Global Interpreter Lock. In combination with the above improvements, these + allow Julia and Python to co-operate on multiple threads. +* Adds method `_jl_call_nogil` to `juliacall.AnyValue` and `juliacall.RawValue` to call + Julia functions with the GIL released. ## 0.9.21 (2024-07-20) * `Serialization.serialize` can use `dill` instead of `pickle` by setting the env var `JULIA_PYTHONCALL_PICKLE=dill`. From 9dbf65e55e3b014871518cc60c14679e3463e172 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 13:56:51 +0100 Subject: [PATCH 36/37] clarification --- docs/src/pythoncall.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index 1c5f3c0c..a7767d42 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -398,7 +398,7 @@ You can also use [multi-threading from Python](@ref py-multi-threading). If Julia's GC collects any Python objects from a thread where the GIL is not currently held, then those Python objects will not immediately be deleted. Instead they will be -queued to be released in a later GC pass. +queued to be deleted in a later GC pass. If you find you have many Python objects not being deleted, you can call -[`PythonCall.GC.gc()`](@ref) or `GC.gc()` (while the GIL is held) to clear the queue. +[`PythonCall.GC.gc()`](@ref) or `GC.gc()` while the GIL is held to clear the queue. From 131c3129d712336a22b5dcddeafb56173be02386 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 3 Aug 2024 18:42:51 +0100 Subject: [PATCH 37/37] rename GIL.release to GIL.unlock and use lock/unlock terminology consistently --- docs/src/juliacall.md | 12 +++++------ docs/src/pythoncall-reference.md | 4 ++-- docs/src/pythoncall.md | 20 +++++++++---------- docs/src/releasenotes.md | 4 ++-- pytest/test_all.py | 2 +- src/GIL/GIL.jl | 34 ++++++++++++++++---------------- src/JlWrap/any.jl | 6 +++--- src/JlWrap/raw.jl | 6 +++--- test/GC.jl | 4 ++-- test/GIL.jl | 12 +++++------ 10 files changed, 52 insertions(+), 52 deletions(-) diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index 9d46ac57..2587aa10 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -132,13 +132,13 @@ caveats. Most importantly, you can only call Python code while Python's [Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is held by the current thread. You can use JuliaCall from any Python thread, and the GIL -will be held whenever any JuliaCall function is used. However, to leverage the benefits -of multi-threading, you can release the GIL while executing any Julia code that does not +is locked by the current thread. You can use JuliaCall from any Python thread, and the GIL +will be locked whenever any JuliaCall function is used. However, to leverage the benefits +of multi-threading, you can unlock the GIL while executing any Julia code that does not interact with Python. The simplest way to do this is using the `_jl_call_nogil` method on Julia functions to -call the function with the GIL released. +call the function with the GIL unlocked. ```python from concurrent.futures import ThreadPoolExecutor, wait @@ -149,14 +149,14 @@ wait(fs) ``` In the above example, we call `Libc.systemsleep(5)` on four threads. Because we -called it with `_jl_call_nogil`, the GIL was released, allowing the threads to run in +called it with `_jl_call_nogil`, the GIL was unlocked, allowing the threads to run in parallel, taking about 5 seconds in total. If we did not use `_jl_call_nogil` (i.e. if we did `pool.submit(jl.Libc.systemsleep, 5)`) then the above code will take 20 seconds because the sleeps run one after another. It is very important that any function called with `_jl_call_nogil` does not interact -with Python at all unless it re-acquires the GIL first, such as by using +with Python at all unless it re-locks the GIL first, such as by using [PythonCall.GIL.@lock](@ref). You can also use [multi-threading from Julia](@ref jl-multi-threading). diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 018cfdfe..b1dd795e 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -226,8 +226,8 @@ See also [`juliacall.AnyValue._jl_call_nogil`](@ref julia-wrappers). ```@docs PythonCall.GIL.lock PythonCall.GIL.@lock -PythonCall.GIL.release -PythonCall.GIL.@release +PythonCall.GIL.unlock +PythonCall.GIL.@unlock PythonCall.GC.gc ``` diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index a7767d42..8b8f19a1 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -370,16 +370,16 @@ caveats. Most importantly, you can only call Python code while Python's [Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is held by the current thread. Ordinarily, the GIL is held by the main thread in Julia, -so if you want to run Python code on any other thread, you must release the GIL from the -main thread and then re-acquire it while running any Python code on other threads. +is locked by the current thread. Ordinarily, the GIL is locked by the main thread in Julia, +so if you want to run Python code on any other thread, you must unlock the GIL from the +main thread and then re-lock it while running any Python code on other threads. -This is made possible by the macros [`PythonCall.GIL.@release`](@ref) and -[`PythonCall.GIL.@lock`](@ref) or the functions [`PythonCall.GIL.release`](@ref) and +This is made possible by the macros [`PythonCall.GIL.@unlock`](@ref) and +[`PythonCall.GIL.@lock`](@ref) or the functions [`PythonCall.GIL.unlock`](@ref) and [`PythonCall.GIL.lock`](@ref) with this pattern: ```julia -PythonCall.GIL.@release Threads.@threads for i in 1:4 +PythonCall.GIL.@unlock Threads.@threads for i in 1:4 PythonCall.GIL.@lock pyimport("time").sleep(5) end ``` @@ -388,17 +388,17 @@ In the above example, we call `time.sleep(5)` four times in parallel. If Julia w started with at least four threads (`julia -t4`) then the above code will take about 5 seconds. -Both `@release` and `@lock` are important. If the GIL were not released, then a deadlock +Both `@unlock` and `@lock` are important. If the GIL were not unlocked, then a deadlock would occur when attempting to lock the already-locked GIL from the threads. If the GIL -were not re-acquired, then Python would crash when interacting with it. +were not re-locked, then Python would crash when interacting with it. You can also use [multi-threading from Python](@ref py-multi-threading). ### Caveat: Garbage collection If Julia's GC collects any Python objects from a thread where the GIL is not currently -held, then those Python objects will not immediately be deleted. Instead they will be +locked, then those Python objects will not immediately be deleted. Instead they will be queued to be deleted in a later GC pass. If you find you have many Python objects not being deleted, you can call -[`PythonCall.GC.gc()`](@ref) or `GC.gc()` while the GIL is held to clear the queue. +[`PythonCall.GC.gc()`](@ref) or `GC.gc()` while the GIL is locked to clear the queue. diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index f5d5eaef..5ce68c15 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -7,11 +7,11 @@ * `GC.disable()` and `GC.enable()` are now a no-op and deprecated since they are no longer required for thread-safety. These will be removed in v1. * Adds `GC.gc()`. -* Adds module `GIL` with `lock()`, `release()`, `@lock` and `@release` for handling the +* Adds module `GIL` with `lock()`, `unlock()`, `@lock` and `@unlock` for handling the Python Global Interpreter Lock. In combination with the above improvements, these allow Julia and Python to co-operate on multiple threads. * Adds method `_jl_call_nogil` to `juliacall.AnyValue` and `juliacall.RawValue` to call - Julia functions with the GIL released. + Julia functions with the GIL unlocked. ## 0.9.21 (2024-07-20) * `Serialization.serialize` can use `dill` instead of `pickle` by setting the env var `JULIA_PYTHONCALL_PICKLE=dill`. diff --git a/pytest/test_all.py b/pytest/test_all.py index c7adac5b..a895398a 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -125,7 +125,7 @@ def test_call_nogil(yld, raw): from time import time from juliacall import Main as jl - # julia implementation of sleep which releases the GIL + # julia implementation of sleep which unlocks the GIL if yld: # use sleep, which yields jsleep = jl.sleep diff --git a/src/GIL/GIL.jl b/src/GIL/GIL.jl index c1e89f23..fb5730bd 100644 --- a/src/GIL/GIL.jl +++ b/src/GIL/GIL.jl @@ -3,7 +3,7 @@ Handling the Python Global Interpreter Lock. -See [`lock`](@ref), [`@lock`](@ref), [`release`](@ref) and [`@release`](@ref). +See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). """ module GIL @@ -12,11 +12,11 @@ using ..C: C """ lock(f) -Acquire the GIL, compute `f()`, release the GIL, then return the result of `f()`. +Unlock the GIL, compute `f()`, unlock the GIL, then return the result of `f()`. Use this to run Python code from threads that do not currently hold the GIL, such as new threads. Since the main Julia thread holds the GIL by default, you will need to -[`release`](@ref) the GIL before using this function. +[`unlock`](@ref) the GIL before using this function. See [`@lock`](@ref) for the macro form. """ @@ -32,11 +32,11 @@ end """ @lock expr -Acquire the GIL, compute `expr`, release the GIL, then return the result of `expr`. +Unlock the GIL, compute `expr`, unlock the GIL, then return the result of `expr`. Use this to run Python code from threads that do not currently hold the GIL, such as new threads. Since the main Julia thread holds the GIL by default, you will need to -[`@release`](@ref) the GIL before using this function. +[`@unlock`](@ref) the GIL before using this function. The macro equivalent of [`lock`](@ref). """ @@ -52,17 +52,17 @@ macro lock(expr) end """ - release(f) + unlock(f) -Release the GIL, compute `f()`, re-acquire the GIL, then return the result of `f()`. +Unlock the GIL, compute `f()`, re-lock the GIL, then return the result of `f()`. -Use this to run non-Python code with the GIL released, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must acquire the GIL using +Use this to run non-Python code with the GIL unlocked, so allowing another thread to run +Python code. That other thread can be a Julia thread, which must lock the GIL using [`lock`](@ref). -See [`@release`](@ref) for the macro form. +See [`@unlock`](@ref) for the macro form. """ -function release(f) +function unlock(f) state = C.PyEval_SaveThread() try f() @@ -72,17 +72,17 @@ function release(f) end """ - @release expr + @unlock expr -Release the GIL, compute `expr`, re-acquire the GIL, then return the result of `expr`. +Unlock the GIL, compute `expr`, re-lock the GIL, then return the result of `expr`. -Use this to run non-Python code with the GIL released, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must acquire the GIL using +Use this to run non-Python code with the GIL unlocked, so allowing another thread to run +Python code. That other thread can be a Julia thread, which must lock the GIL using [`@lock`](@ref). -The macro equivalent of [`release`](@ref). +The macro equivalent of [`unlock`](@ref). """ -macro release(expr) +macro unlock(expr) quote state = C.PyEval_SaveThread() try diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index 573ca1a4..8ea73907 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -55,12 +55,12 @@ function pyjlany_call_nogil(self, args_::Py, kwargs_::Py) if pylen(kwargs_) > 0 args = pyconvert(Vector{Any}, args_) kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = Py(GIL.@release self(args...; kwargs...)) + ans = Py(GIL.@unlock self(args...; kwargs...)) elseif pylen(args_) > 0 args = pyconvert(Vector{Any}, args_) - ans = Py(GIL.@release self(args...)) + ans = Py(GIL.@unlock self(args...)) else - ans = Py(GIL.@release self()) + ans = Py(GIL.@unlock self()) end pydel!(args_) pydel!(kwargs_) diff --git a/src/JlWrap/raw.jl b/src/JlWrap/raw.jl index 21e0fa7e..501f2aef 100644 --- a/src/JlWrap/raw.jl +++ b/src/JlWrap/raw.jl @@ -44,12 +44,12 @@ function pyjlraw_call_nogil(self, args_::Py, kwargs_::Py) if pylen(kwargs_) > 0 args = pyconvert(Vector{Any}, args_) kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = pyjlraw(GIL.@release self(args...; kwargs...)) + ans = pyjlraw(GIL.@unlock self(args...; kwargs...)) elseif pylen(args_) > 0 args = pyconvert(Vector{Any}, args_) - ans = pyjlraw(GIL.@release self(args...)) + ans = pyjlraw(GIL.@unlock self(args...)) else - ans = pyjlraw(GIL.@release self()) + ans = pyjlraw(GIL.@unlock self()) end pydel!(args_) pydel!(kwargs_) diff --git a/test/GC.jl b/test/GC.jl index cc8d6ff0..84aa8477 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1,7 +1,7 @@ @testitem "GC.gc()" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@release Threads.@threads for obj in pyobjs + PythonCall.GIL.@unlock Threads.@threads for obj in pyobjs finalize(obj) end end @@ -13,7 +13,7 @@ end @testitem "GC.GCHook" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@release Threads.@threads for obj in pyobjs + PythonCall.GIL.@unlock Threads.@threads for obj in pyobjs finalize(obj) end end diff --git a/test/GIL.jl b/test/GIL.jl index a0249282..ca1f6405 100644 --- a/test/GIL.jl +++ b/test/GIL.jl @@ -1,8 +1,8 @@ -@testitem "release and lock" begin - # This calls Python's time.sleep(1) twice concurrently. Since sleep() releases the +@testitem "unlock and lock" begin + # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.release() do + PythonCall.GIL.unlock() do Threads.@threads for i = 1:2 PythonCall.GIL.lock() do pyimport("time").sleep(1) @@ -20,11 +20,11 @@ end end -@testitem "@release and @lock" begin - # This calls Python's time.sleep(1) twice concurrently. Since sleep() releases the +@testitem "@unlock and @lock" begin + # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.@release Threads.@threads for i = 1:2 + PythonCall.GIL.@unlock Threads.@threads for i = 1:2 PythonCall.GIL.@lock pyimport("time").sleep(1) end end