Auto gil

CondaPkg.toml

@@ -13,7 +13,7 @@ build = "**cpython**"
 version = ">=3.10,!=3.14.0,!=3.14.1,<4"
 
 [dev.deps]
-matplotlib = ""
-numpy = ""
-pyside6 = ""
-pandas = ""
+# matplotlib = ""
+# numpy = ""
+# pyside6 = ""
+# pandas = ""

src/API/publics.jl

@@ -2,7 +2,6 @@ if Base.VERSION ≥ v"1.11"
     eval(Meta.parse("""
         public
             GC,
-            GIL,
             VERSION,
 
             # C

src/C/C.jl

@@ -26,6 +26,7 @@ include("pointers.jl")
 include("extras.jl")
 include("context.jl")
 include("api.jl")
+include("threadstate.jl")
 
 function __init__()
     init_context()

src/C/context.jl

@@ -18,6 +18,7 @@ A handle to a loaded instance of libpython, its interpreter, function pointers,
     which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown
     version::Union{VersionNumber,Missing} = missing
     is_free_threaded::Bool = false
+    thread_state::Ptr{Cvoid} = C_NULL
 end
 
 const CTX = Context()
@@ -30,79 +31,6 @@ function _atpyexit()
     return
 end
 
-
-function setup_onfixedthread()
-    channel_input = Channel(1)
-    channel_output = Channel(1)
-    islaunched = Ref(false) # use Ref to avoid closure boxing of variable
-    function launch_worker(tid)
-        islaunched[] && error("Cannot launch more than once: call setup_onfixedthread again if need be.")
-        islaunched[] = true
-        worker_task = Task() do
-            while true
-                f = take!(channel_input)
-                ret = try
-                    Some(invokelatest(f))
-                    # invokelatest is necessary for development and interactive use.
-                    # Otherwise, only a method f defined in a world prior to the call of
-                    # launch_worker would work.
-                catch e
-                    e, catch_backtrace()
-                end
-                put!(channel_output, ret)
-            end
-        end
-        # code adapted from set_task_tid! in StableTasks.jl, itself taken from Dagger.jl
-        worker_task.sticky = true
-        for _ in 1:100
-            # try to fix the task id to tid, retrying up to 100 times
-            ret = ccall(:jl_set_task_tid, Cint, (Any, Cint), worker_task, tid-1)
-            if ret == 1
-                break # success
-            elseif ret == 0
-                yield()
-            else
-                error("Unexpected retcode from jl_set_task_tid: $ret")
-            end
-        end
-        if Threads.threadid(worker_task) != tid
-            error("Failed setting the thread ID to $tid.")
-        end
-        schedule(worker_task)
-    end
-    function onfixedthread(f)
-        put!(channel_input, f)
-        ret = take!(channel_output)
-        if ret isa Tuple
-            e, backtrace = ret
-            printstyled(stderr, "ERROR: "; color=:red, bold=true)
-            showerror(stderr, e)
-            Base.show_backtrace(stderr, backtrace)
-            println(stderr)
-            throw(e) # the stacktrace of the actual error is printed above
-        else
-            something(ret)
-        end
-    end
-    launch_worker, onfixedthread
-end
-
-# launch_on_main_thread is used in init_context(), after which on_main_thread becomes usable
-const launch_on_main_thread, on_main_thread = setup_onfixedthread()
-
-"""
-    on_main_thread(f)
-
-Execute `f()` on the main thread.
-
-!!! warning
-    The value returned by `on_main_thread(f)` cannot be type-inferred by the compiler:
-    if necessary, use explicit type annotations such as `on_main_thread(f)::T`, where `T` is
-    the expected return type.
-"""
-on_main_thread
-
-
 function init_context()
 
     CTX.is_embedded = haskey(ENV, "__JULIA_PYTHONCALL_EMBEDDED_LIBPTR__")
@@ -280,17 +208,13 @@ function init_context()
 
             # Start the interpreter and register exit hooks
             Py_InitializeEx(0)
-            atexit() do
-                CTX.is_initialized = false
-                if Py_FinalizeEx() == -1
-                    @warn "Py_FinalizeEx() error"
-                end
-            end
+            atexit(_atjlexit)
         end
         CTX.is_initialized = true
         if Py_AtExit(@cfunction(_atpyexit, Cvoid, ())) == -1
             @warn "Py_AtExit() error"
         end
+        CTX.thread_state = PyEval_SaveThread()
     end
 
     # HACK: If we are using CondaPkg, prevent child processes from using it by explicitly
@@ -318,8 +242,6 @@ function init_context()
     )
     CTX.is_free_threaded = occursin("free-threading build", verstr)
 
-    launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable
-
     @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded
 
     return
@@ -334,9 +256,3 @@ function Base.show(io::IO, ::MIME"text/plain", ctx::Context)
         show(io, getfield(ctx, k))
     end
 end
-
-const PYTHONCALL_UUID = Base.UUID("6099a3de-0909-46bc-b1f4-468b9a2dfc0d")
-const PYTHONCALL_PKGID = Base.PkgId(PYTHONCALL_UUID, "PythonCall")
-
-const PYCALL_UUID = Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0")
-const PYCALL_PKGID = Base.PkgId(PYCALL_UUID, "PyCall")

src/C/pointers.jl

@@ -23,6 +23,11 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     :PyGILState_Release => (PyGILState_STATE,) => Cvoid,
     :PyGILState_GetThisThreadState => () => Ptr{Cvoid},
     :PyGILState_Check => () => Cint,
+    :PyThreadState_Get => () => Ptr{Cvoid},
+    :PyThreadState_New => (Ptr{Cvoid},) => Ptr{Cvoid},
+    :PyThreadState_Swap => (Ptr{Cvoid},) => Ptr{Cvoid},
+    :PyThreadState_GetUnchecked => () => Ptr{Cvoid},
+    :PyInterpreterState_Main => () => Ptr{Cvoid},
     # IMPORT
     :PyImport_ImportModule => (Ptr{Cchar},) => PyPtr,
     :PyImport_Import => (PyPtr,) => PyPtr,
@@ -282,18 +287,28 @@ end
 
 const POINTERS = CAPIPointers()
 
-@eval init_pointers(p::CAPIPointers = POINTERS, lib::Ptr = CTX.lib_ptr) = begin
+@eval function init_pointers(p::CAPIPointers = POINTERS, lib::Ptr = CTX.lib_ptr)
+    # get the function pointers
     $([
         :(p.$name = dlsym(lib, $(QuoteNode(name))))
         for name in CAPI_FUNCS
+        if name != :PyThreadState_GetUnchecked
     ]...)
+    # PyThreadState_GetUnchecked was called _PyThreadState_UncheckedGet on 3.5 - 3.12
+    p.PyThreadState_GetUnchecked = dlsym_e(lib, :PyThreadState_GetUnchecked)
+    if p.PyThreadState_GetUnchecked == C_NULL
+        p.PyThreadState_GetUnchecked = dlsym(lib, :_PyThreadState_UncheckedGet)
+    end
+    # get the exception pointers
     $(
         [
             :(p.$name =
                     Base.unsafe_load(Ptr{PyPtr}(dlsym(lib, $(QuoteNode(name)))::Ptr))) for name in CAPI_EXCEPTIONS
         ]...
     )
+    # get other object pointers
     $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...)
+    # get the PyOS_InputHook pointer
     p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook)
 end
 

src/C/threadstate.jl

@@ -0,0 +1,55 @@
+const THREAD_STATE_LOCK = ReentrantLock()
+const THREAD_STATE_LOCK_PER_THREAD = OncePerThread{ReentrantLock}(ReentrantLock)
+const THREAD_STATE = OncePerThread{Ptr{Cvoid}}(() -> PyThreadState_New(PyInterpreterState_Main()))
+
+get_thread_state_lock() = CTX.is_free_threaded ? THREAD_STATE_LOCK_PER_THREAD() : THREAD_STATE_LOCK
+get_thread_state() = THREAD_STATE()
+
+"""
+    @withts ex
+
+Run the given expression `ex` with an attached CPython thread-state.
+
+Limitations:
+- This uses a `ReentrantLock` for co-operation with other Julia tasks so cannot be
+  called in finalizers.
+"""
+macro withts(ex)
+    quote
+        # task must be sticky to prevent the thread from changing during this block
+        task = current_task()
+        sticky = task.sticky
+        task.sticky = true
+        # acquire a re-entrant lock, so that no other task can set the thread state
+        thelock = get_thread_state_lock()
+        lock(thelock)
+        # attach the python thread state. this blocks the thread until the thread state
+        # is detached, hence the above lock, so that the blocking is co-operative with
+        # other julia tasks, rather than just hanging the thread. since the lock is
+        # re-entrant, the task might enter this locked area again while still locked
+        # (that is, nesting this macro is allowed) so we use PyThreadState_Swap, which
+        # will return NULL in the outermost invocation, and will return THREAD_STATE()
+        # in all the innermost ones.
+        tstate = get_thread_state()
+        tstate_prev = PyThreadState_Swap(tstate)
+        # run the desired expression
+        try
+            $(esc(ex))
+        finally
+            # swap the threadstate back to its prior value
+            PyThreadState_Swap(tstate_prev)
+            # reset the task stickiness, so that a previously non-sticky task remains non-
+            # sticky and can be migrated outside of this block
+            task.sticky = sticky
+            # unlock, to allow another task to call into python
+            unlock(thelock)
+        end
+    end
+end
+
+function _atjlexit()
+    CTX.is_initialized = false
+    if @withts Py_FinalizeEx() == -1
+        @warn "Py_FinalizeEx() error"
+    end
+end

src/Convert/ctypes.jl

@@ -1,7 +1,7 @@
 struct pyconvert_rule_ctypessimplevalue{R,S} <: Function end
 
 function (::pyconvert_rule_ctypessimplevalue{R,SAFE})(::Type{T}, x::Py) where {R,SAFE,T}
-    Base.GC.@preserve x begin
+    C.@withts Base.GC.@preserve x begin
         ptr = C.PySimpleObject_GetValue(Ptr{R}, x)
         ans = unsafe_load(ptr)
         if SAFE

src/Convert/numpy.jl

@@ -1,7 +1,7 @@
 struct pyconvert_rule_numpysimplevalue{R,S} <: Function end
 
 function (::pyconvert_rule_numpysimplevalue{R,SAFE})(::Type{T}, x::Py) where {R,SAFE,T}
-    ans = C.PySimpleObject_GetValue(R, x)
+    ans = C.@withts C.PySimpleObject_GetValue(R, x)
     if SAFE
         pyconvert_return(convert(T, ans))
     else
@@ -10,15 +10,15 @@ function (::pyconvert_rule_numpysimplevalue{R,SAFE})(::Type{T}, x::Py) where {R,
 end
 
 function pyconvert_rule_datetime64(::Type{DateTime64}, x::Py)
-    pyconvert_return(C.PySimpleObject_GetValue(DateTime64, x))
+    pyconvert_return(C.@withts C.PySimpleObject_GetValue(DateTime64, x))
 end
 
 function pyconvert_rule_datetime64(::Type{T}, x::Py) where {T<:InlineDateTime64}
-    pyconvert_tryconvert(T, C.PySimpleObject_GetValue(DateTime64, x))
+    pyconvert_tryconvert(T, C.@withts C.PySimpleObject_GetValue(DateTime64, x))
 end
 
 function pyconvert_rule_datetime64(::Type{T}, x::Py) where {T<:NumpyDates.DatesInstant}
-    d = C.PySimpleObject_GetValue(DateTime64, x)
+    d = C.@withts C.PySimpleObject_GetValue(DateTime64, x)
     if isnan(d)
         pyconvert_unconverted()
     else
@@ -27,7 +27,7 @@ function pyconvert_rule_datetime64(::Type{T}, x::Py) where {T<:NumpyDates.DatesI
 end
 
 function pyconvert_rule_datetime64(::Type{Missing}, x::Py)
-    d = C.PySimpleObject_GetValue(DateTime64, x)
+    d = C.@withts C.PySimpleObject_GetValue(DateTime64, x)
     if isnan(d)
         pyconvert_return(missing)
     else
@@ -36,7 +36,7 @@ function pyconvert_rule_datetime64(::Type{Missing}, x::Py)
 end
 
 function pyconvert_rule_datetime64(::Type{Nothing}, x::Py)
-    d = C.PySimpleObject_GetValue(DateTime64, x)
+    d = C.@withts C.PySimpleObject_GetValue(DateTime64, x)
     if isnan(d)
         pyconvert_return(nothing)
     else
@@ -45,15 +45,15 @@ function pyconvert_rule_datetime64(::Type{Nothing}, x::Py)
 end
 
 function pyconvert_rule_timedelta64(::Type{TimeDelta64}, x::Py)
-    pyconvert_return(C.PySimpleObject_GetValue(TimeDelta64, x))
+    pyconvert_return(C.@withts C.PySimpleObject_GetValue(TimeDelta64, x))
 end
 
 function pyconvert_rule_timedelta64(::Type{T}, x::Py) where {T<:InlineTimeDelta64}
-    pyconvert_tryconvert(T, C.PySimpleObject_GetValue(TimeDelta64, x))
+    pyconvert_tryconvert(T, C.@withts C.PySimpleObject_GetValue(TimeDelta64, x))
 end
 
 function pyconvert_rule_timedelta64(::Type{T}, x::Py) where {T<:NumpyDates.DatesPeriod}
-    d = C.PySimpleObject_GetValue(TimeDelta64, x)
+    d = C.@withts C.PySimpleObject_GetValue(TimeDelta64, x)
     if isnan(d)
         pyconvert_unconverted()
     else
@@ -62,7 +62,7 @@ function pyconvert_rule_timedelta64(::Type{T}, x::Py) where {T<:NumpyDates.Dates
 end
 
 function pyconvert_rule_timedelta64(::Type{Missing}, x::Py)
-    d = C.PySimpleObject_GetValue(TimeDelta64, x)
+    d = C.@withts C.PySimpleObject_GetValue(TimeDelta64, x)
     if isnan(d)
         pyconvert_return(missing)
     else
@@ -71,7 +71,7 @@ function pyconvert_rule_timedelta64(::Type{Missing}, x::Py)
 end
 
 function pyconvert_rule_timedelta64(::Type{Nothing}, x::Py)
-    d = C.PySimpleObject_GetValue(TimeDelta64, x)
+    d = C.@withts C.PySimpleObject_GetValue(TimeDelta64, x)
     if isnan(d)
         pyconvert_return(missing)
     else

src/Convert/pyconvert.jl

@@ -230,7 +230,7 @@ function _pyconvert_get_rules(pytype::Py)
         end
     end
     for (t, x) in reverse(collect(zip(mro, xmro)))
-        if C.PyType_CheckBuffer(t)
+        if C.@withts C.PyType_CheckBuffer(t)
             push!(x, "<buffer>")
             break
         end
@@ -345,7 +345,7 @@ function pytryconvert(::Type{T}, x_) where {T}
     tptr = C.Py_Type(x)
     trules = pyconvert_rules_cache(T)
     rules = get!(trules, tptr) do
-        t = pynew(incref(tptr))
+        t = C.@withts pynew(incref(tptr))
         ans = pyconvert_get_rules(T, t)::Vector{Function}
         unsafe_pydel(t)
         ans