diff --git a/CondaPkg.toml b/CondaPkg.toml index 555e81b3..5ec6ce0d 100644 --- a/CondaPkg.toml +++ b/CondaPkg.toml @@ -13,7 +13,7 @@ build = "**cpython**" version = ">=3.10,!=3.14.0,!=3.14.1,<4" [dev.deps] -matplotlib = "" -numpy = "" -pyside6 = "" -pandas = "" +# matplotlib = "" +# numpy = "" +# pyside6 = "" +# pandas = "" diff --git a/src/API/publics.jl b/src/API/publics.jl index bda0fc47..965f959e 100644 --- a/src/API/publics.jl +++ b/src/API/publics.jl @@ -2,7 +2,6 @@ if Base.VERSION ≥ v"1.11" eval(Meta.parse(""" public GC, - GIL, VERSION, # C diff --git a/src/C/C.jl b/src/C/C.jl index f7b4e020..dae3df6b 100644 --- a/src/C/C.jl +++ b/src/C/C.jl @@ -26,6 +26,7 @@ include("pointers.jl") include("extras.jl") include("context.jl") include("api.jl") +include("threadstate.jl") function __init__() init_context() diff --git a/src/C/context.jl b/src/C/context.jl index 6ccdab89..cd591ac2 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -18,6 +18,7 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown version::Union{VersionNumber,Missing} = missing is_free_threaded::Bool = false + thread_state::Ptr{Cvoid} = C_NULL end const CTX = Context() @@ -30,79 +31,6 @@ function _atpyexit() return end - -function setup_onfixedthread() - channel_input = Channel(1) - channel_output = Channel(1) - islaunched = Ref(false) # use Ref to avoid closure boxing of variable - function launch_worker(tid) - islaunched[] && error("Cannot launch more than once: call setup_onfixedthread again if need be.") - islaunched[] = true - worker_task = Task() do - while true - f = take!(channel_input) - ret = try - Some(invokelatest(f)) - # invokelatest is necessary for development and interactive use. - # Otherwise, only a method f defined in a world prior to the call of - # launch_worker would work. - catch e - e, catch_backtrace() - end - put!(channel_output, ret) - end - end - # code adapted from set_task_tid! in StableTasks.jl, itself taken from Dagger.jl - worker_task.sticky = true - for _ in 1:100 - # try to fix the task id to tid, retrying up to 100 times - ret = ccall(:jl_set_task_tid, Cint, (Any, Cint), worker_task, tid-1) - if ret == 1 - break # success - elseif ret == 0 - yield() - else - error("Unexpected retcode from jl_set_task_tid: $ret") - end - end - if Threads.threadid(worker_task) != tid - error("Failed setting the thread ID to $tid.") - end - schedule(worker_task) - end - function onfixedthread(f) - put!(channel_input, f) - ret = take!(channel_output) - if ret isa Tuple - e, backtrace = ret - printstyled(stderr, "ERROR: "; color=:red, bold=true) - showerror(stderr, e) - Base.show_backtrace(stderr, backtrace) - println(stderr) - throw(e) # the stacktrace of the actual error is printed above - else - something(ret) - end - end - launch_worker, onfixedthread -end - -# launch_on_main_thread is used in init_context(), after which on_main_thread becomes usable -const launch_on_main_thread, on_main_thread = setup_onfixedthread() - -""" - on_main_thread(f) - -Execute `f()` on the main thread. - -!!! warning - The value returned by `on_main_thread(f)` cannot be type-inferred by the compiler: - if necessary, use explicit type annotations such as `on_main_thread(f)::T`, where `T` is - the expected return type. -""" -on_main_thread - - function init_context() CTX.is_embedded = haskey(ENV, "__JULIA_PYTHONCALL_EMBEDDED_LIBPTR__") @@ -280,17 +208,13 @@ function init_context() # Start the interpreter and register exit hooks Py_InitializeEx(0) - atexit() do - CTX.is_initialized = false - if Py_FinalizeEx() == -1 - @warn "Py_FinalizeEx() error" - end - end + atexit(_atjlexit) end CTX.is_initialized = true if Py_AtExit(@cfunction(_atpyexit, Cvoid, ())) == -1 @warn "Py_AtExit() error" end + CTX.thread_state = PyEval_SaveThread() end # HACK: If we are using CondaPkg, prevent child processes from using it by explicitly @@ -318,8 +242,6 @@ function init_context() ) CTX.is_free_threaded = occursin("free-threading build", verstr) - launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable - @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded return @@ -334,9 +256,3 @@ function Base.show(io::IO, ::MIME"text/plain", ctx::Context) show(io, getfield(ctx, k)) end end - -const PYTHONCALL_UUID = Base.UUID("6099a3de-0909-46bc-b1f4-468b9a2dfc0d") -const PYTHONCALL_PKGID = Base.PkgId(PYTHONCALL_UUID, "PythonCall") - -const PYCALL_UUID = Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0") -const PYCALL_PKGID = Base.PkgId(PYCALL_UUID, "PyCall") diff --git a/src/C/pointers.jl b/src/C/pointers.jl index 9644329f..375d135c 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -23,6 +23,11 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( :PyGILState_Release => (PyGILState_STATE,) => Cvoid, :PyGILState_GetThisThreadState => () => Ptr{Cvoid}, :PyGILState_Check => () => Cint, + :PyThreadState_Get => () => Ptr{Cvoid}, + :PyThreadState_New => (Ptr{Cvoid},) => Ptr{Cvoid}, + :PyThreadState_Swap => (Ptr{Cvoid},) => Ptr{Cvoid}, + :PyThreadState_GetUnchecked => () => Ptr{Cvoid}, + :PyInterpreterState_Main => () => Ptr{Cvoid}, # IMPORT :PyImport_ImportModule => (Ptr{Cchar},) => PyPtr, :PyImport_Import => (PyPtr,) => PyPtr, @@ -282,18 +287,28 @@ end const POINTERS = CAPIPointers() -@eval init_pointers(p::CAPIPointers = POINTERS, lib::Ptr = CTX.lib_ptr) = begin +@eval function init_pointers(p::CAPIPointers = POINTERS, lib::Ptr = CTX.lib_ptr) + # get the function pointers $([ :(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_FUNCS + if name != :PyThreadState_GetUnchecked ]...) + # PyThreadState_GetUnchecked was called _PyThreadState_UncheckedGet on 3.5 - 3.12 + p.PyThreadState_GetUnchecked = dlsym_e(lib, :PyThreadState_GetUnchecked) + if p.PyThreadState_GetUnchecked == C_NULL + p.PyThreadState_GetUnchecked = dlsym(lib, :_PyThreadState_UncheckedGet) + end + # get the exception pointers $( [ :(p.$name = Base.unsafe_load(Ptr{PyPtr}(dlsym(lib, $(QuoteNode(name)))::Ptr))) for name in CAPI_EXCEPTIONS ]... ) + # get other object pointers $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...) + # get the PyOS_InputHook pointer p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook) end diff --git a/src/C/threadstate.jl b/src/C/threadstate.jl new file mode 100644 index 00000000..634b8275 --- /dev/null +++ b/src/C/threadstate.jl @@ -0,0 +1,55 @@ +const THREAD_STATE_LOCK = ReentrantLock() +const THREAD_STATE_LOCK_PER_THREAD = OncePerThread{ReentrantLock}(ReentrantLock) +const THREAD_STATE = OncePerThread{Ptr{Cvoid}}(() -> PyThreadState_New(PyInterpreterState_Main())) + +get_thread_state_lock() = CTX.is_free_threaded ? THREAD_STATE_LOCK_PER_THREAD() : THREAD_STATE_LOCK +get_thread_state() = THREAD_STATE() + +""" + @withts ex + +Run the given expression `ex` with an attached CPython thread-state. + +Limitations: +- This uses a `ReentrantLock` for co-operation with other Julia tasks so cannot be + called in finalizers. +""" +macro withts(ex) + quote + # task must be sticky to prevent the thread from changing during this block + task = current_task() + sticky = task.sticky + task.sticky = true + # acquire a re-entrant lock, so that no other task can set the thread state + thelock = get_thread_state_lock() + lock(thelock) + # attach the python thread state. this blocks the thread until the thread state + # is detached, hence the above lock, so that the blocking is co-operative with + # other julia tasks, rather than just hanging the thread. since the lock is + # re-entrant, the task might enter this locked area again while still locked + # (that is, nesting this macro is allowed) so we use PyThreadState_Swap, which + # will return NULL in the outermost invocation, and will return THREAD_STATE() + # in all the innermost ones. + tstate = get_thread_state() + tstate_prev = PyThreadState_Swap(tstate) + # run the desired expression + try + $(esc(ex)) + finally + # swap the threadstate back to its prior value + PyThreadState_Swap(tstate_prev) + # reset the task stickiness, so that a previously non-sticky task remains non- + # sticky and can be migrated outside of this block + task.sticky = sticky + # unlock, to allow another task to call into python + unlock(thelock) + end + end +end + +function _atjlexit() + CTX.is_initialized = false + if @withts Py_FinalizeEx() == -1 + @warn "Py_FinalizeEx() error" + end +end diff --git a/src/Convert/ctypes.jl b/src/Convert/ctypes.jl index d18bbc74..50dfbf09 100644 --- a/src/Convert/ctypes.jl +++ b/src/Convert/ctypes.jl @@ -1,7 +1,7 @@ struct pyconvert_rule_ctypessimplevalue{R,S} <: Function end function (::pyconvert_rule_ctypessimplevalue{R,SAFE})(::Type{T}, x::Py) where {R,SAFE,T} - Base.GC.@preserve x begin + C.@withts Base.GC.@preserve x begin ptr = C.PySimpleObject_GetValue(Ptr{R}, x) ans = unsafe_load(ptr) if SAFE diff --git a/src/Convert/numpy.jl b/src/Convert/numpy.jl index 73383d54..bfae33ea 100644 --- a/src/Convert/numpy.jl +++ b/src/Convert/numpy.jl @@ -1,7 +1,7 @@ struct pyconvert_rule_numpysimplevalue{R,S} <: Function end function (::pyconvert_rule_numpysimplevalue{R,SAFE})(::Type{T}, x::Py) where {R,SAFE,T} - ans = C.PySimpleObject_GetValue(R, x) + ans = C.@withts C.PySimpleObject_GetValue(R, x) if SAFE pyconvert_return(convert(T, ans)) else @@ -10,15 +10,15 @@ function (::pyconvert_rule_numpysimplevalue{R,SAFE})(::Type{T}, x::Py) where {R, end function pyconvert_rule_datetime64(::Type{DateTime64}, x::Py) - pyconvert_return(C.PySimpleObject_GetValue(DateTime64, x)) + pyconvert_return(C.@withts C.PySimpleObject_GetValue(DateTime64, x)) end function pyconvert_rule_datetime64(::Type{T}, x::Py) where {T<:InlineDateTime64} - pyconvert_tryconvert(T, C.PySimpleObject_GetValue(DateTime64, x)) + pyconvert_tryconvert(T, C.@withts C.PySimpleObject_GetValue(DateTime64, x)) end function pyconvert_rule_datetime64(::Type{T}, x::Py) where {T<:NumpyDates.DatesInstant} - d = C.PySimpleObject_GetValue(DateTime64, x) + d = C.@withts C.PySimpleObject_GetValue(DateTime64, x) if isnan(d) pyconvert_unconverted() else @@ -27,7 +27,7 @@ function pyconvert_rule_datetime64(::Type{T}, x::Py) where {T<:NumpyDates.DatesI end function pyconvert_rule_datetime64(::Type{Missing}, x::Py) - d = C.PySimpleObject_GetValue(DateTime64, x) + d = C.@withts C.PySimpleObject_GetValue(DateTime64, x) if isnan(d) pyconvert_return(missing) else @@ -36,7 +36,7 @@ function pyconvert_rule_datetime64(::Type{Missing}, x::Py) end function pyconvert_rule_datetime64(::Type{Nothing}, x::Py) - d = C.PySimpleObject_GetValue(DateTime64, x) + d = C.@withts C.PySimpleObject_GetValue(DateTime64, x) if isnan(d) pyconvert_return(nothing) else @@ -45,15 +45,15 @@ function pyconvert_rule_datetime64(::Type{Nothing}, x::Py) end function pyconvert_rule_timedelta64(::Type{TimeDelta64}, x::Py) - pyconvert_return(C.PySimpleObject_GetValue(TimeDelta64, x)) + pyconvert_return(C.@withts C.PySimpleObject_GetValue(TimeDelta64, x)) end function pyconvert_rule_timedelta64(::Type{T}, x::Py) where {T<:InlineTimeDelta64} - pyconvert_tryconvert(T, C.PySimpleObject_GetValue(TimeDelta64, x)) + pyconvert_tryconvert(T, C.@withts C.PySimpleObject_GetValue(TimeDelta64, x)) end function pyconvert_rule_timedelta64(::Type{T}, x::Py) where {T<:NumpyDates.DatesPeriod} - d = C.PySimpleObject_GetValue(TimeDelta64, x) + d = C.@withts C.PySimpleObject_GetValue(TimeDelta64, x) if isnan(d) pyconvert_unconverted() else @@ -62,7 +62,7 @@ function pyconvert_rule_timedelta64(::Type{T}, x::Py) where {T<:NumpyDates.Dates end function pyconvert_rule_timedelta64(::Type{Missing}, x::Py) - d = C.PySimpleObject_GetValue(TimeDelta64, x) + d = C.@withts C.PySimpleObject_GetValue(TimeDelta64, x) if isnan(d) pyconvert_return(missing) else @@ -71,7 +71,7 @@ function pyconvert_rule_timedelta64(::Type{Missing}, x::Py) end function pyconvert_rule_timedelta64(::Type{Nothing}, x::Py) - d = C.PySimpleObject_GetValue(TimeDelta64, x) + d = C.@withts C.PySimpleObject_GetValue(TimeDelta64, x) if isnan(d) pyconvert_return(missing) else diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 10435f11..a1a398a9 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -230,7 +230,7 @@ function _pyconvert_get_rules(pytype::Py) end end for (t, x) in reverse(collect(zip(mro, xmro))) - if C.PyType_CheckBuffer(t) + if C.@withts C.PyType_CheckBuffer(t) push!(x, "") break end @@ -345,7 +345,7 @@ function pytryconvert(::Type{T}, x_) where {T} tptr = C.Py_Type(x) trules = pyconvert_rules_cache(T) rules = get!(trules, tptr) do - t = pynew(incref(tptr)) + t = C.@withts pynew(incref(tptr)) ans = pyconvert_get_rules(T, t)::Vector{Function} unsafe_pydel(t) ans diff --git a/src/Convert/rules.jl b/src/Convert/rules.jl index 43e029b8..47012fd2 100644 --- a/src/Convert/rules.jl +++ b/src/Convert/rules.jl @@ -58,45 +58,47 @@ pyconvert_rule_bytes(::Type{Base.CodeUnits{UInt8,String}}, x::Py) = ### int -pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = begin - # first try to convert to Clonglong (or Culonglong if unsigned) - v = - T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : - C.PyLong_AsLongLong(x) - if !iserrset_ambig(v) - # success - return pyconvert_tryconvert(T, v) - elseif errmatches(pybuiltins.OverflowError) - # overflows Clonglong or Culonglong - errclear() - if T in ( - Bool, - Int8, - Int16, - Int32, - Int64, - Int128, - UInt8, - UInt16, - UInt32, - UInt64, - UInt128, - ) && - typemin(typeof(v)) ≤ typemin(T) && - typemax(T) ≤ typemax(typeof(v)) - # definitely overflows S, give up now - return pyconvert_unconverted() - else - # try converting -> int -> str -> BigInt -> T - x_int = pyint(x) - x_str = pystr(String, x_int) - unsafe_pydel(x_int) - v = parse(BigInt, x_str) +function pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} + C.@withts begin + # first try to convert to Clonglong (or Culonglong if unsigned) + v = + T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : + C.PyLong_AsLongLong(x) + if !iserrset_ambig(v) + # success return pyconvert_tryconvert(T, v) + elseif errmatches(pybuiltins.OverflowError) + # overflows Clonglong or Culonglong + errclear() + if T in ( + Bool, + Int8, + Int16, + Int32, + Int64, + Int128, + UInt8, + UInt16, + UInt32, + UInt64, + UInt128, + ) && + typemin(typeof(v)) ≤ typemin(T) && + typemax(T) ≤ typemax(typeof(v)) + # definitely overflows S, give up now + return pyconvert_unconverted() + else + # try converting -> int -> str -> BigInt -> T + x_int = pyint(x) + x_str = pystr(String, x_int) + unsafe_pydel(x_int) + v = parse(BigInt, x_str) + return pyconvert_tryconvert(T, v) + end + else + # other error + pythrow() end - else - # other error - pythrow() end end diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 50ff0eb2..a90f4a6b 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -87,7 +87,7 @@ it at some indeterminate point in the future. function unsafe_pydel(x::Py) ptr = getptr(x) if ptr != C.PyNULL - C.Py_DecRef(ptr) + C.@withts C.Py_DecRef(ptr) setptr!(x, C.PyNULL) end return @@ -270,7 +270,7 @@ Base.hasproperty(x::Py, k::String) = pyhasattr(x, k) Base.setproperty!(x::Py, k::Symbol, v) = pysetattr(x, string(k), v) Base.setproperty!(x::Py, k::String, v) = pysetattr(x, k, v) -function _propertynames(x::Py, private::Bool) +function Base.propertynames(x::Py, private::Bool = false) # this follows the logic of rlcompleter.py function classmembers(c) r = pydir(c) @@ -291,16 +291,6 @@ function _propertynames(x::Py, private::Bool) return Symbol[Symbol(pystr_asstring(word)) for word in words] end -function Base.propertynames(x::Py, private::Bool = false) - if C.PyGILState_Check() == 1 - _propertynames(x, private) - else - C.on_main_thread() do - _propertynames(x, private) - end::Vector{Symbol} - end -end - Base.Bool(x::Py) = pytruth(x) Base.length(x::Py) = pylen(x) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 1bb634cb..be29f76e 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -14,7 +14,7 @@ pyisnot(x, y) = !pyis(x, y) Equivalent to `repr(x)` in Python. """ -pyrepr(x) = pynew(errcheck(@autopy x C.PyObject_Repr(x_))) +pyrepr(x) = C.@withts pynew(errcheck(@autopy x C.PyObject_Repr(x_))) pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ @@ -22,7 +22,7 @@ pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pyde Equivalent to `ascii(x)` in Python. """ -pyascii(x) = pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) +pyascii(x) = C.@withts pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) pyascii(::Type{String}, x) = (s = pyascii(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ @@ -33,20 +33,21 @@ Equivalent to `hasattr(x, k)` in Python. Tests if `getattr(x, k)` raises an `AttributeError`. """ function pyhasattr(x, k) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return false + C.@withts begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end -# pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(x_, k_)) == 1 """ pygetattr(x, k, [d]) @@ -55,18 +56,20 @@ Equivalent to `getattr(x, k)` or `x.k` in Python. If `d` is specified, it is returned if the attribute does not exist. """ -pygetattr(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) +pygetattr(x, k) = C.@withts pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) function pygetattr(x, k, d) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return d + C.@withts begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -75,56 +78,56 @@ end Equivalent to `setattr(x, k, v)` or `x.k = v` in Python. """ -pysetattr(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) +pysetattr(x, k, v) = C.@withts (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) """ pydelattr(x, k) Equivalent to `delattr(x, k)` or `del x.k` in Python. """ -pydelattr(x, k) = (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) +pydelattr(x, k) = C.@withts (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) """ pyissubclass(s, t) Test if `s` is a subclass of `t`. Equivalent to `issubclass(s, t)` in Python. """ -pyissubclass(s, t) = errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 +pyissubclass(s, t) = C.@withts errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 """ pyisinstance(x, t) Test if `x` is of type `t`. Equivalent to `isinstance(x, t)` in Python. """ -pyisinstance(x, t) = errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 +pyisinstance(x, t) = C.@withts errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 """ pyhash(x) Equivalent to `hash(x)` in Python, converted to an `Integer`. """ -pyhash(x) = errcheck(@autopy x C.PyObject_Hash(x_)) +pyhash(x) = C.@withts errcheck(@autopy x C.PyObject_Hash(x_)) """ pytruth(x) The truthyness of `x`. Equivalent to `bool(x)` in Python, converted to a `Bool`. """ -pytruth(x) = errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 +pytruth(x) = C.@withts errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 """ pynot(x) The falsyness of `x`. Equivalent to `not x` in Python, converted to a `Bool`. """ -pynot(x) = errcheck(@autopy x C.PyObject_Not(x_)) == 1 +pynot(x) = C.@withts errcheck(@autopy x C.PyObject_Not(x_)) == 1 """ pylen(x) The length of `x`. Equivalent to `len(x)` in Python, converted to an `Integer`. """ -pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) +pylen(x) = C.@withts errcheck(@autopy x C.PyObject_Length(x_)) """ pyhasitem(x, k) @@ -132,17 +135,19 @@ pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) Test if `pygetitem(x, k)` raises a `KeyError` or `AttributeError`. """ function pyhasitem(x, k) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return false + C.@withts begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end @@ -154,18 +159,20 @@ Equivalent `x[k]` in Python. If `d` is specified, it is returned if the item does not exist (i.e. if `x[k]` raises a `KeyError` or `IndexError`). """ -pygetitem(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) +pygetitem(x, k) = C.@withts pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) function pygetitem(x, k, d) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return d + C.@withts begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -174,26 +181,26 @@ end Equivalent to `setitem(x, k, v)` or `x[k] = v` in Python. """ -pysetitem(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) +pysetitem(x, k, v) = C.@withts (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) """ pydelitem(x, k) Equivalent to `delitem(x, k)` or `del x[k]` in Python. """ -pydelitem(x, k) = (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) +pydelitem(x, k) = C.@withts (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) """ pydir(x) Equivalent to `dir(x)` in Python. """ -pydir(x) = pynew(errcheck(@autopy x C.PyObject_Dir(x_))) +pydir(x) = C.@withts pynew(errcheck(@autopy x C.PyObject_Dir(x_))) -pycallargs(f) = pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) -pycallargs(f, args) = pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) +pycallargs(f) = C.@withts pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) +pycallargs(f, args) = C.@withts pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) pycallargs(f, args, kwargs) = - pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) + C.@withts pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) """ pycall(f, args...; kwargs...) @@ -223,7 +230,7 @@ pycall(f, args...; kwargs...) = Equivalent to `x == y` in Python. The second form converts to `Bool`. """ -pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) +pyeq(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) """ pyne(x, y) @@ -231,7 +238,7 @@ pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ)) Equivalent to `x != y` in Python. The second form converts to `Bool`. """ -pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) +pyne(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) """ pyle(x, y) @@ -239,7 +246,7 @@ pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE)) Equivalent to `x <= y` in Python. The second form converts to `Bool`. """ -pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) +pyle(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) """ pylt(x, y) @@ -247,7 +254,7 @@ pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE)) Equivalent to `x < y` in Python. The second form converts to `Bool`. """ -pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) +pylt(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) """ pyge(x, y) @@ -255,7 +262,7 @@ pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT)) Equivalent to `x >= y` in Python. The second form converts to `Bool`. """ -pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) +pyge(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) """ pygt(x, y) @@ -263,26 +270,27 @@ pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE)) Equivalent to `x > y` in Python. The second form converts to `Bool`. """ -pygt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) +pygt(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) + pyeq(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 + C.@withts errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 pyne(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 + C.@withts errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 pyle(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 + C.@withts errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 pylt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 + C.@withts errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 pyge(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 + C.@withts errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 pygt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 + C.@withts errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 """ pycontains(x, v) Equivalent to `v in x` in Python. """ -pycontains(x, v) = errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 +pycontains(x, v) = C.@withts errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 """ pyin(v, x) @@ -301,31 +309,31 @@ pynotin(v, x) = !pyin(v, x) Equivalent to `-x` in Python. """ -pyneg(x) = pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) +pyneg(x) = C.@withts pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) """ pypos(x) Equivalent to `+x` in Python. """ -pypos(x) = pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) +pypos(x) = C.@withts pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) """ pyabs(x) Equivalent to `abs(x)` in Python. """ -pyabs(x) = pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) +pyabs(x) = C.@withts pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) """ pyinv(x) Equivalent to `~x` in Python. """ -pyinv(x) = pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) +pyinv(x) = C.@withts pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) """ pyindex(x) Convert `x` losslessly to an `int`. """ -pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) +pyindex(x) = C.@withts pynew(errcheck(@autopy x C.PyNumber_Index(x_))) # binary """ @@ -333,79 +341,79 @@ pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) Equivalent to `x + y` in Python. """ -pyadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) +pyadd(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) """ pysub(x, y) Equivalent to `x - y` in Python. """ -pysub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) +pysub(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) """ pymul(x, y) Equivalent to `x * y` in Python. """ -pymul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) +pymul(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) """ pymatmul(x, y) Equivalent to `x @ y` in Python. """ -pymatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) +pymatmul(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) """ pyfloordiv(x, y) Equivalent to `x // y` in Python. """ -pyfloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) +pyfloordiv(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) """ pytruediv(x, y) Equivalent to `x / y` in Python. """ -pytruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) +pytruediv(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) """ pymod(x, y) Equivalent to `x % y` in Python. """ -pymod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) +pymod(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) """ pydivmod(x, y) Equivalent to `divmod(x, y)` in Python. """ -pydivmod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) +pydivmod(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) """ pylshift(x, y) Equivalent to `x << y` in Python. """ -pylshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) +pylshift(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) """ pyrshift(x, y) Equivalent to `x >> y` in Python. """ -pyrshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) +pyrshift(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) """ pyand(x, y) Equivalent to `x & y` in Python. """ -pyand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) +pyand(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) """ pyxor(x, y) Equivalent to `x ^ y` in Python. """ -pyxor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) +pyxor(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) """ pyor(x, y) Equivalent to `x | y` in Python. """ -pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) +pyor(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) # binary in-place """ @@ -413,73 +421,73 @@ pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) In-place add. `x = pyiadd(x, y)` is equivalent to `x += y` in Python. """ -pyiadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) +pyiadd(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) """ pyisub(x, y) In-place subtract. `x = pyisub(x, y)` is equivalent to `x -= y` in Python. """ -pyisub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) +pyisub(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) """ pyimul(x, y) In-place multiply. `x = pyimul(x, y)` is equivalent to `x *= y` in Python. """ -pyimul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) +pyimul(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) """ pyimatmul(x, y) In-place matrix multiply. `x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. """ -pyimatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) +pyimatmul(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) """ pyifloordiv(x, y) In-place floor divide. `x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. """ -pyifloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) +pyifloordiv(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) """ pyitruediv(x, y) In-place true division. `x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. """ -pyitruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) +pyitruediv(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) """ pyimod(x, y) In-place subtraction. `x = pyimod(x, y)` is equivalent to `x %= y` in Python. """ -pyimod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) +pyimod(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) """ pyilshift(x, y) In-place left shift. `x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. """ -pyilshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) +pyilshift(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) """ pyirshift(x, y) In-place right shift. `x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. """ -pyirshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) +pyirshift(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) """ pyiand(x, y) In-place and. `x = pyiand(x, y)` is equivalent to `x &= y` in Python. """ -pyiand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) +pyiand(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) """ pyixor(x, y) In-place xor. `x = pyixor(x, y)` is equivalent to `x ^= y` in Python. """ -pyixor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) +pyixor(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) """ pyior(x, y) In-place or. `x = pyior(x, y)` is equivalent to `x |= y` in Python. """ -pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) +pyior(x, y) = C.@withts pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) # power """ @@ -488,14 +496,14 @@ pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) Equivalent to `x ** y` or `pow(x, y, z)` in Python. """ pypow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) + C.@withts pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) """ pyipow(x, y, z=None) In-place power. `x = pyipow(x, y)` is equivalent to `x **= y` in Python. """ pyipow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) + C.@withts pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) ### iter @@ -504,7 +512,7 @@ pyipow(x, y, z = pybuiltins.None) = Equivalent to `iter(x)` in Python. """ -pyiter(x) = pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) +pyiter(x) = C.@withts pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) """ pynext(x, [d]) @@ -515,18 +523,22 @@ Returns the next item from the iterator `x`. If there are no more items, returns given, else raises `StopIteration`. """ function pynext(x) - ptr = errcheck_ambig(C.PyIter_Next(x)) - if ptr == C.PyNULL - errset(pybuiltins.StopIteration) - pythrow() - else - pynew(ptr) + C.@withts begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + if ptr == C.PyNULL + errset(pybuiltins.StopIteration) + pythrow() + else + pynew(ptr) + end end end function pynext(x, d) - ptr = errcheck_ambig(C.PyIter_Next(x)) - ptr == C.PyNULL ? d : pynew(ptr) + C.@withts begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr == C.PyNULL ? d : pynew(ptr) + end end """ @@ -534,7 +546,7 @@ end Return the next item in the iterator `x`. When there are no more items, return NULL. """ -unsafe_pynext(x::Py) = Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) +unsafe_pynext(x::Py) = C.@withts Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) ### None @@ -567,15 +579,15 @@ end ### str -pystr_fromUTF8(x::Ptr, n::Integer) = pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) -pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) +pystr_fromUTF8(x::Ptr, n::Integer) = C.@withts pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) +pystr_fromUTF8(x) = Base.GC.@preserve x pystr_fromUTF8(pointer(x), sizeof(x)) """ pystr(x) Convert `x` to a Python `str`. """ -pystr(x) = pynew(errcheck(@autopy x C.PyObject_Str(x_))) +pystr(x) = C.@withts pynew(errcheck(@autopy x C.PyObject_Str(x_))) pystr(x::String) = pystr_fromUTF8(x) pystr(x::SubString{String}) = pystr_fromUTF8(x) pystr(x::Char) = pystr(string(x)) @@ -583,31 +595,33 @@ pystr(x::AbstractString) = pystr(convert(String, x)::String) pystr(x::AbstractChar) = pystr(convert(Char, x)::Char) pystr(::Type{String}, x) = (s = pystr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) -pystr_asUTF8bytes(x::Py) = pynew(errcheck(C.PyUnicode_AsUTF8String(x))) +pystr_asUTF8bytes(x::Py) = C.@withts pynew(errcheck(C.PyUnicode_AsUTF8String(x))) pystr_asUTF8vector(x::Py) = (b = pystr_asUTF8bytes(x); ans = pybytes_asvector(b); unsafe_pydel(b); ans) pystr_asstring(x::Py) = (b = pystr_asUTF8bytes(x); ans = pybytes_asUTF8string(b); unsafe_pydel(b); ans) function pystr_intern!(x::Py) - ptr = Ref(getptr(x)) - C.PyUnicode_InternInPlace(ptr) - setptr!(x, ptr[]) + C.@withts Base.GC.@preserve x begin + ptr = Ref(getptr(x)) + C.PyUnicode_InternInPlace(ptr) + setptr!(x, ptr[]) + end end pyisstr(x) = pytypecheckfast(x, C.Py_TPFLAGS_UNICODE_SUBCLASS) ### bytes -pybytes_fromdata(x::Ptr, n::Integer) = pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) -pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) +pybytes_fromdata(x::Ptr, n::Integer) = C.@withts pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) +pybytes_fromdata(x) = Base.GC.@preserve x pybytes_fromdata(pointer(x), sizeof(x)) """ pybytes(x) Convert `x` to a Python `bytes`. """ -pybytes(x) = pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) +pybytes(x) = C.@withts pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) pybytes(x::Vector{UInt8}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,String}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,SubString{String}}) = pybytes_fromdata(x) @@ -621,7 +635,7 @@ pyisbytes(x) = pytypecheckfast(x, C.Py_TPFLAGS_BYTES_SUBCLASS) function pybytes_asdata(x::Py) ptr = Ref(Ptr{Cchar}(0)) len = Ref(C.Py_ssize_t(0)) - errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) + C.@withts errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) ptr[], len[] end @@ -639,7 +653,7 @@ end pyint_fallback( x::Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}, -) = pynew(errcheck(C.PyLong_FromString(string(x, base = 32), C_NULL, 32))) +) = C.@withts pynew(errcheck(C.PyLong_FromString(string(x, base = 32), C_NULL, 32))) pyint_fallback(x::Integer) = pyint_fallback(BigInt(x)) """ @@ -650,7 +664,7 @@ Convert `x` to a Python `int`. function pyint(x::Integer = 0) y = mod(x, Clonglong) if x == y - pynew(errcheck(C.PyLong_FromLongLong(y))) + C.@withts pynew(errcheck(C.PyLong_FromLongLong(y))) else pyint_fallback(x) end @@ -658,12 +672,12 @@ end function pyint(x::Unsigned) y = mod(x, Culonglong) if x == y - pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) + C.@withts pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) else pyint_fallback(x) end end -pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(x_))) +pyint(x) = C.@withts @autopy x pynew(errcheck(C.PyNumber_Long(x_))) pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) @@ -674,12 +688,12 @@ pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) Convert `x` to a Python `float`. """ -pyfloat(x::Real = 0.0) = pynew(errcheck(C.PyFloat_FromDouble(x))) -pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(x_))) +pyfloat(x::Real = 0.0) = C.@withts pynew(errcheck(C.PyFloat_FromDouble(x))) +pyfloat(x) = C.@withts @autopy x pynew(errcheck(C.PyNumber_Float(x_))) pyisfloat(x) = pytypecheck(x, pybuiltins.float) -pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) +pyfloat_asdouble(x) = C.@withts errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) ### complex @@ -689,7 +703,7 @@ pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) Convert `x` to a Python `complex`, or create one from given real and imaginary parts. """ -pycomplex(x::Real = 0.0, y::Real = 0.0) = pynew(errcheck(C.PyComplex_FromDoubles(x, y))) +pycomplex(x::Real = 0.0, y::Real = 0.0) = C.@withts pynew(errcheck(C.PyComplex_FromDoubles(x, y))) pycomplex(x::Complex) = pycomplex(real(x), imag(x)) pycomplex(x) = pybuiltins.complex(x) pycomplex(x, y) = pybuiltins.complex(x, y) @@ -697,9 +711,11 @@ pycomplex(x, y) = pybuiltins.complex(x, y) pyiscomplex(x) = pytypecheck(x, pybuiltins.complex) function pycomplex_ascomplex(x) - c = @autopy x C.PyComplex_AsCComplex(x_) - c.real == -1 && c.imag == 0 && errcheck() - return Complex(c.real, c.imag) + C.@withts begin + c = @autopy x C.PyComplex_AsCComplex(x_) + c.real == -1 && c.imag == 0 && errcheck() + Complex(c.real, c.imag) + end end ### type @@ -709,7 +725,7 @@ end The Python `type` of `x`. """ -pytype(x) = pynew(errcheck(@autopy x C.PyObject_Type(x_))) +pytype(x) = C.@withts pynew(errcheck(@autopy x C.PyObject_Type(x_))) """ pytype(name, bases, dict) @@ -786,7 +802,7 @@ end pyistype(x) = pytypecheckfast(x, C.Py_TPFLAGS_TYPE_SUBCLASS) -pytypecheck(x, t) = (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 +pytypecheck(x, t) = C.@withts (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 ### slice @@ -797,7 +813,7 @@ pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 Construct a Python `slice`. Unspecified arguments default to `None`. """ pyslice(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) + C.@withts pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) pyslice(y) = pyslice(pybuiltins.None, y, pybuiltins.None) pyisslice(x) = pytypecheck(x, pybuiltins.slice) @@ -819,15 +835,15 @@ pyisrange(x) = pytypecheck(x, pybuiltins.range) ### tuple -pynulltuple(len) = pynew(errcheck(C.PyTuple_New(len))) +pynulltuple(len) = C.@withts pynew(errcheck(C.PyTuple_New(len))) function pytuple_setitem(xs::Py, i, x) - errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) + C.@withts errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) return xs end function pytuple_getitem(xs::Py, i) - Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) + C.@withts Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) end function pytuple_fromiter(xs) @@ -874,16 +890,16 @@ pyistuple(x) = pytypecheckfast(x, C.Py_TPFLAGS_TUPLE_SUBCLASS) ### list -pynulllist(len) = pynew(errcheck(C.PyList_New(len))) +pynulllist(len) = C.@withts pynew(errcheck(C.PyList_New(len))) function pylist_setitem(xs::Py, i, x) - errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) + C.@withts errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) return xs end -pylist_append(xs::Py, x) = errcheck(@autopy x C.PyList_Append(xs, x_)) +pylist_append(xs::Py, x) = C.@withts errcheck(@autopy x C.PyList_Append(xs, x_)) -pylist_astuple(x) = pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) +pylist_astuple(x) = C.@withts pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) function pylist_fromiter(xs) sz = Base.IteratorSize(typeof(xs)) @@ -953,7 +969,7 @@ end ### set -pyset_add(set::Py, x) = (errcheck(@autopy x C.PySet_Add(set, x_)); set) +pyset_add(set::Py, x) = C.@withts (errcheck(@autopy x C.PySet_Add(set, x_)); set) function pyset_update_fromiter(set::Py, xs) for x in xs @@ -972,7 +988,7 @@ Convert `x` to a Python `set`. If `x` is a Python object, this is equivalent to `set(x)` in Python. Otherwise `x` must be iterable. """ -pyset() = pynew(errcheck(C.PySet_New(C.PyNULL))) +pyset() = C.@withts pynew(errcheck(C.PySet_New(C.PyNULL))) pyset(x) = ispy(x) ? pybuiltins.set(x) : pyset_fromiter(x) """ @@ -983,12 +999,12 @@ Convert `x` to a Python `frozenset`. If `x` is a Python object, this is equivalent to `frozenset(x)` in Python. Otherwise `x` must be iterable. """ -pyfrozenset() = pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) +pyfrozenset() = C.@withts pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) ### dict -pydict_setitem(x::Py, k, v) = errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) +pydict_setitem(x::Py, k, v) = C.@withts errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) function pydict_fromiter(kvs) ans = pydict() @@ -1017,7 +1033,7 @@ If `x` is a Python object, this is equivalent to `dict(x)` in Python. Otherwise `x` must iterate over key-value pairs. """ pydict(; kwargs...) = - isempty(kwargs) ? pynew(errcheck(C.PyDict_New())) : pystrdict_fromiter(kwargs) + isempty(kwargs) ? (C.@withts pynew(errcheck(C.PyDict_New()))) : pystrdict_fromiter(kwargs) pydict(x) = ispy(x) ? pybuiltins.dict(x) : pydict_fromiter(x) pydict(x::NamedTuple) = pydict(; x...) pydict(pair::Pair, pairs::Pair...) = pydict((pair, pairs...)) @@ -1471,7 +1487,7 @@ Import a module `m`, or an attribute `k`, or a tuple of attributes. If several arguments are given, return the results of importing each one in a tuple. """ -pyimport(m) = pynew(errcheck(@autopy m C.PyImport_Import(m_))) +pyimport(m) = C.@withts pynew(errcheck(@autopy m C.PyImport_Import(m_))) pyimport((m, k)::Pair) = (m_ = pyimport(m); k_ = pygetattr(m_, k); unsafe_pydel(m_); k_) pyimport((m, ks)::Pair{<:Any,<:Tuple}) = (m_ = pyimport(m); ks_ = map(k -> pygetattr(m_, k), ks); unsafe_pydel(m_); ks_) diff --git a/src/Core/err.jl b/src/Core/err.jl index 4ee00c74..4e1f288b 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -64,19 +64,21 @@ end function Base.getproperty(exc::PyException, k::Symbol) if k in (:t, :v, :b) && !exc._isnormalized - errnormalize!(exc._t, exc._v, exc._b) - pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) - pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) - pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) - pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) - exc._isnormalized = true + C.@withts begin + errnormalize!(exc._t, exc._v, exc._b) + pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) + pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) + pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) + pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) + exc._isnormalized = true + end end k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end pythrow() = throw(PyException(errget()..., false)) -file_to_pymodule(fname::String) = begin +function file_to_pymodule(fname::String) isfile(fname) || return nothing modules = pyimport("sys").modules for (n, m) in modules.items() diff --git a/src/Core/juliacall.jl b/src/Core/juliacall.jl index 6a9db0e7..65317297 100644 --- a/src/Core/juliacall.jl +++ b/src/Core/juliacall.jl @@ -31,5 +31,5 @@ function init_juliacall() @assert !pybool_asbool(jl.CONFIG["init"]) end pycopy!(pyJuliaError, jl.JuliaError) - CPyExc_JuliaError[] = incref(getptr(pyJuliaError)) + CPyExc_JuliaError[] = C.@withts incref(getptr(pyJuliaError)) end diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 67c43565..c7d1ff79 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -23,56 +23,14 @@ end const QUEUE = (; items = C.PyPtr[], lock = Threads.SpinLock()) const HOOK = Ref{WeakRef}() -""" - PythonCall.GC.disable() - -Do nothing. - -!!! note - - Historically this would disable the PythonCall garbage collector. This was required - for safety in multi-threaded code but is no longer needed, so this is now a no-op. -""" -function disable() - Base.depwarn( - "disabling the PythonCall GC is no longer needed for thread-safety", - :disable, - ) - nothing -end - -""" - PythonCall.GC.enable() - -Do nothing. - -!!! note - - Historically this would enable the PythonCall garbage collector. This was required - for safety in multi-threaded code but is no longer needed, so this is now a no-op. -""" -function enable() - Base.depwarn( - "disabling the PythonCall GC is no longer needed for thread-safety", - :enable, - ) - nothing -end - """ PythonCall.GC.gc() Free any Python objects waiting to be freed. - -These are objects that were finalized from a thread that was not holding the Python -GIL at the time. - -Like most PythonCall functions, this must only be called from the main thread (i.e. the -thread currently holding the Python GIL.) """ function gc() if C.CTX.is_initialized - unsafe_free_queue() + C.@withts unsafe_free_queue() end nothing end @@ -94,8 +52,8 @@ function enqueue(ptr::C.PyPtr) # If C.CTX.is_initialized is false then the Python interpreter hasn't started yet # or has been finalized; either way attempting to free will cause an error. if ptr != C.PyNULL && C.CTX.is_initialized - if C.PyGILState_Check() == 1 - # If the current thread holds the GIL, then we can immediately free. + if C.PyThreadState_GetUnchecked() != C_NULL + # If there is an attached thread-state, then we can immediately free. C.Py_DecRef(ptr) # We may as well also free any other enqueued objects. if !isempty(QUEUE.items) @@ -115,7 +73,7 @@ end function enqueue_all(ptrs) if any(!=(C.PyNULL), ptrs) && C.CTX.is_initialized - if C.PyGILState_Check() == 1 + if C.PyThreadState_GetUnchecked() != C_NULL for ptr in ptrs if ptr != C.PyNULL C.Py_DecRef(ptr) @@ -150,7 +108,7 @@ end function _gchook_finalizer(x) if C.CTX.is_initialized finalizer(_gchook_finalizer, x) - if !isempty(QUEUE.items) && C.PyGILState_Check() == 1 + if !isempty(QUEUE.items) && C.PyThreadState_GetUnchecked() != C_NULL unsafe_free_queue() end end diff --git a/src/GIL/GIL.jl b/src/GIL/GIL.jl deleted file mode 100644 index f4b386ce..00000000 --- a/src/GIL/GIL.jl +++ /dev/null @@ -1,129 +0,0 @@ -""" - module PythonCall.GIL - -Handling the Python Global Interpreter Lock. - -See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). - -!!! warning - - Multi-threading support is experimental and can change without notice. -""" -module GIL - -using ..C: C - -if Base.VERSION ≥ v"1.11" - eval( - Expr( - :public, - :lock, - Symbol("@lock"), - :unlock, - Symbol("@unlock"), - ), - ) -end - - -""" - lock(f) - -Lock the GIL, compute `f()`, unlock the GIL, then return the result of `f()`. - -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`unlock`](@ref) the GIL before using this function. - -See [`@lock`](@ref) for the macro form. - -!!! warning - - This function is experimental. Its semantics may be changed without notice. -""" -function lock(f) - state = C.PyGILState_Ensure() - try - f() - finally - C.PyGILState_Release(state) - end -end - -""" - @lock expr - -Lock the GIL, compute `expr`, unlock the GIL, then return the result of `expr`. - -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`@unlock`](@ref) the GIL before using this function. - -The macro equivalent of [`lock`](@ref). - -!!! warning - - This macro is experimental. Its semantics may be changed without notice. -""" -macro lock(expr) - quote - state = C.PyGILState_Ensure() - try - $(esc(expr)) - finally - C.PyGILState_Release(state) - end - end -end - -""" - unlock(f) - -Unlock the GIL, compute `f()`, re-lock the GIL, then return the result of `f()`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`lock`](@ref). - -See [`@unlock`](@ref) for the macro form. - -!!! warning - - This function is experimental. Its semantics may be changed without notice. -""" -function unlock(f) - state = C.PyEval_SaveThread() - try - f() - finally - C.PyEval_RestoreThread(state) - end -end - -""" - @unlock expr - -Unlock the GIL, compute `expr`, re-lock the GIL, then return the result of `expr`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`@lock`](@ref). - -The macro equivalent of [`unlock`](@ref). - -!!! warning - - This macro is experimental. Its semantics may be changed without notice. -""" -macro unlock(expr) - quote - state = C.PyEval_SaveThread() - try - $(esc(expr)) - finally - C.PyEval_RestoreThread(state) - end - end -end - -end diff --git a/src/PythonCall.jl b/src/PythonCall.jl index c6b166bd..c1dabb39 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -6,26 +6,25 @@ include("API/API.jl") include("Utils/Utils.jl") include("NumpyDates/NumpyDates.jl") include("C/C.jl") -include("GIL/GIL.jl") include("GC/GC.jl") include("Core/Core.jl") include("Convert/Convert.jl") include("PyMacro/PyMacro.jl") include("Wrap/Wrap.jl") -include("JlWrap/JlWrap.jl") -include("Compat/Compat.jl") +# include("JlWrap/JlWrap.jl") +# include("Compat/Compat.jl") -# not API but used in tests -for k in [ - :pyjlanytype, - :pyjlarraytype, - :pyjlvectortype, - :pyjlbinaryiotype, - :pyjltextiotype, - :pyjldicttype, - :pyjlsettype, -] - @eval using .JlWrap: $k -end +# # not API but used in tests +# for k in [ +# :pyjlanytype, +# :pyjlarraytype, +# :pyjlvectortype, +# :pyjlbinaryiotype, +# :pyjltextiotype, +# :pyjldicttype, +# :pyjlsettype, +# ] +# @eval using .JlWrap: $k +# end end diff --git a/src/Wrap/PyArray.jl b/src/Wrap/PyArray.jl index 64bb3176..e4f8f355 100644 --- a/src/Wrap/PyArray.jl +++ b/src/Wrap/PyArray.jl @@ -52,7 +52,7 @@ function pyarray_make( @debug "failed to make PyArray from __array_interface__" exc = exc end end - if buffer && C.PyObject_CheckBuffer(x) + if buffer && C.@withts C.PyObject_CheckBuffer(x) try return pyarray_make(A, x, PyArraySource_Buffer(x)) catch exc @@ -166,7 +166,7 @@ function PyArraySource_ArrayInterface(x::Py, d::Py = x.__array_interface__) else memview = @py memoryview(data === None ? x : data) unsafe_pydel(data) - buf = UnsafePtr(C.PyMemoryView_GET_BUFFER(memview)) + buf = UnsafePtr(C.@withts C.PyMemoryView_GET_BUFFER(memview)) ptr = buf.buf[!] readonly = buf.readonly[] != 0 handle = Py((x, memview)) @@ -361,8 +361,8 @@ struct PyArraySource_ArrayStruct <: PyArraySource info::C.PyArrayInterface end function PyArraySource_ArrayStruct(x::Py, capsule::Py = x.__array_struct__) - name = C.PyCapsule_GetName(capsule) - ptr = C.PyCapsule_GetPointer(capsule, name) + name = C.@withts C.PyCapsule_GetName(capsule) + ptr = C.@withts C.PyCapsule_GetPointer(capsule, name) info = unsafe_load(Ptr{C.PyArrayInterface}(ptr)) @assert info.two == 2 return PyArraySource_ArrayStruct(x, capsule, info) @@ -439,7 +439,7 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) return Utils.StaticString{UInt32,div(size, 4)} elseif kind == 86 # V = void (should have descr) hasdescr || error("not supported: void dtype with no descr") - descr = pynew(incref(src.info.descr)) + descr = C.@withts pynew(incref(src.info.descr)) T = pyarray_descr_to_type(descr) sizeof(T) == size || error("size mismatch: itemsize=$size but sizeof(descr)=$(sizeof(T))") @@ -490,7 +490,7 @@ struct PyArraySource_Buffer <: PyArraySource end function PyArraySource_Buffer(x::Py) memview = pybuiltins.memoryview(x) - buf = C.UnsafePtr(C.PyMemoryView_GET_BUFFER(memview)) + buf = C.UnsafePtr(C.@withts C.PyMemoryView_GET_BUFFER(memview)) buf.suboffsets[] == C_NULL || error("PyArray does not support buffers with non-trivial suboffsets (PIL-style indirect layout)") PyArraySource_Buffer(x, memview, buf) @@ -647,7 +647,7 @@ function pyarray_load(::Type{T}, p::Ptr{R}) where {T,R} unsafe_load(p) elseif R == C.PyPtr u = unsafe_load(p) - o = u == C_NULL ? pynew(Py(nothing)) : pynew(incref(u)) + o = C.@withts u == C_NULL ? pynew(Py(nothing)) : pynew(incref(u)) T == Py ? o : pyconvert(T, o) else convert(T, unsafe_load(p)) @@ -658,7 +658,7 @@ function pyarray_store!(::Type{T}, p::Ptr{R}, x::T) where {R,T} if R == T unsafe_store!(p, x) elseif R == C.PyPtr - @autopy x begin + C.@withts @autopy x begin decref(unsafe_load(p).ptr) unsafe_store!(p, getptr(incref(x_))) end